worklyn / zukapublic
Agent-first git hosting. One Rust binary: git over HTTP and SSH, a REST API, MCP, CI, and multi-tenant isolation.
Get a copy:
git clone https://zuka.worklyn.com/worklyn/zuka.git
| 1 | // Provisioning a tenant. |
| 2 | // |
| 3 | // The container operations sit behind a trait for one reason: they are the only part |
| 4 | // of the control plane that cannot be tested without a Linux host running Incus. |
| 5 | // Everything above this line — the state machine, the reconciler, routing, the |
| 6 | // signed assertion — is exercised against `FakeProvisioner` on any machine, and the |
| 7 | // Incus implementation stays a thin, reviewable shim over the CLI. |
| 8 | // |
| 9 | // The methods are synchronous. Each is a subprocess spawn, so they run through |
| 10 | // `git::exec::blocking` at the call site, matching how the rest of the service |
| 11 | // treats blocking work. |
| 12 | |
| 13 | use crate::error::{Error, Result}; |
| 14 | use crate::git::validate::Name; |
| 15 | use serde::{Deserialize, Serialize}; |
| 16 | |
| 17 | /// Where a tenant is in its lifecycle. |
| 18 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 19 | #[serde(rename_all = "snake_case")] |
| 20 | pub enum State { |
| 21 | /// Requested. The reconciler will drive it forward. |
| 22 | Provisioning, |
| 23 | /// Running and reachable. |
| 24 | Ready, |
| 25 | /// Stopped to save resources; started again on demand. |
| 26 | Stopped, |
| 27 | /// Provisioning failed. `detail` says why. |
| 28 | Failed, |
| 29 | } |
| 30 | |
| 31 | /// A provisioned tenant. |
| 32 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 33 | pub struct Instance { |
| 34 | /// Address the control plane proxies to. |
| 35 | pub endpoint: String, |
| 36 | pub state: State, |
| 37 | } |
| 38 | |
| 39 | pub trait Provisioner: Send + Sync { |
| 40 | /// Create and start a tenant. Must be idempotent: the reconciler retries. |
| 41 | fn create(&self, account: &Name) -> Result<Instance>; |
| 42 | fn delete(&self, account: &Name) -> Result<()>; |
| 43 | fn status(&self, account: &Name) -> Result<Option<Instance>>; |
| 44 | /// Replace the binary in a running tenant and restart it. |
| 45 | /// |
| 46 | /// Separate from `create` because it must not relaunch the container: the |
| 47 | /// tenant's repositories live on its own disk, and recreating it to change a |
| 48 | /// binary would destroy them. |
| 49 | fn upgrade(&self, account: &Name) -> Result<()>; |
| 50 | fn start(&self, account: &Name) -> Result<Instance>; |
| 51 | fn stop(&self, account: &Name) -> Result<()>; |
| 52 | } |
| 53 | |
| 54 | /// Instance name for an account. Incus names are DNS-ish, and our account names are |
| 55 | /// already validated to a stricter set, so the prefix is the only transformation. |
| 56 | pub fn instance_name(account: &Name) -> String { |
| 57 | format!("{}-{}", crate::brand::NAME, account.as_str()) |
| 58 | } |
| 59 | |
| 60 | // ── Incus ────────────────────────────────────────────────────────────────── |
| 61 | |
| 62 | /// Drives Incus through its CLI. |
| 63 | /// |
| 64 | /// The CLI rather than the REST socket: it is the documented interface, it handles |
| 65 | /// authentication against a remote host, and it keeps this shim small enough to read |
| 66 | /// in one sitting — which matters because it is the part with the least test |
| 67 | /// coverage. |
| 68 | pub struct IncusProvisioner { |
| 69 | /// Image to launch, e.g. `images:debian/12`. |
| 70 | image: String, |
| 71 | /// Network the container joins. |
| 72 | network: String, |
| 73 | /// Port the tenant listens on inside its container. |
| 74 | port: u16, |
| 75 | /// Where the tenant binary lives on the host, to push into the container. |
| 76 | /// |
| 77 | /// Must be statically linked: the host and the container image do not share a |
| 78 | /// C library, and a dynamically linked binary fails at exec inside the |
| 79 | /// container with a glibc version error. |
| 80 | binary: String, |
| 81 | /// Public half of the control plane's signing key, so the tenant can verify |
| 82 | /// the assertions it is sent. Public by construction — it forges nothing. |
| 83 | control_public_key: String, |
| 84 | /// What the outside world calls this service. |
| 85 | /// |
| 86 | /// A tenant is only ever reached through the control plane, so it cannot know |
| 87 | /// its own public address — left to itself it advertises its bind address, and |
| 88 | /// a user is handed `http://0.0.0.0:8790/...` as a clone URL. |
| 89 | public: PublicAddresses, |
| 90 | } |
| 91 | |
| 92 | /// The three origins a tenant may advertise. Grouped so adding a public surface |
| 93 | /// does not turn the provisioner constructor into an argument parade. |
| 94 | #[derive(Debug, Clone, Default)] |
| 95 | pub struct PublicAddresses { |
| 96 | pub url: Option<String>, |
| 97 | pub ssh: Option<String>, |
| 98 | /// Separate package-only origin. A tenant needs the value to advertise |
| 99 | /// source-module imports, even though Caddy terminates the public host. |
| 100 | pub package: Option<String>, |
| 101 | } |
| 102 | |
| 103 | impl IncusProvisioner { |
| 104 | pub fn new( |
| 105 | image: String, |
| 106 | network: String, |
| 107 | port: u16, |
| 108 | binary: String, |
| 109 | control_public_key: String, |
| 110 | public: PublicAddresses, |
| 111 | ) -> Self { |
| 112 | IncusProvisioner { |
| 113 | image, |
| 114 | network, |
| 115 | port, |
| 116 | binary, |
| 117 | control_public_key, |
| 118 | public, |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | /// The tenant's environment file, written inside the container. |
| 123 | fn tenant_env(&self) -> String { |
| 124 | let p = crate::brand::env_prefix(); |
| 125 | let mut lines = vec![ |
| 126 | format!("{p}MODE=tenant"), |
| 127 | format!("{p}DATA_DIR=/var/lib/{}", crate::brand::NAME), |
| 128 | format!("{p}BIND=0.0.0.0:{}", self.port), |
| 129 | // SSH terminates at the control plane; a tenant never listens for it. |
| 130 | format!("{p}SSH_BIND=off"), |
| 131 | format!("{p}CONTROL_PUBLIC_KEYS={}", self.control_public_key), |
| 132 | format!("{p}CI_ENABLED=1"), |
| 133 | ]; |
| 134 | |
| 135 | // Without these the tenant advertises its own bind address, and a caller is |
| 136 | // handed a clone URL pointing at a container they cannot reach. |
| 137 | if let Some(url) = &self.public.url { |
| 138 | lines.push(format!("{p}PUBLIC_URL={url}")); |
| 139 | } |
| 140 | if let Some(ssh) = &self.public.ssh { |
| 141 | lines.push(format!("{p}PUBLIC_SSH={ssh}")); |
| 142 | } |
| 143 | if let Some(url) = &self.public.package { |
| 144 | lines.push(format!("{p}PACKAGE_URL={url}")); |
| 145 | } |
| 146 | lines.join("\n") + "\n" |
| 147 | } |
| 148 | |
| 149 | /// The systemd unit that runs the tenant inside its container. |
| 150 | fn tenant_unit(&self) -> String { |
| 151 | let name = crate::brand::NAME; |
| 152 | format!( |
| 153 | "[Unit]\n\ |
| 154 | Description={name} tenant\n\ |
| 155 | After=network-online.target\n\ |
| 156 | \n\ |
| 157 | [Service]\n\ |
| 158 | ExecStart=/usr/local/bin/{name}\n\ |
| 159 | EnvironmentFile=/etc/{name}.env\n\ |
| 160 | Restart=always\n\ |
| 161 | RestartSec=2\n\ |
| 162 | \n\ |
| 163 | [Install]\n\ |
| 164 | WantedBy=multi-user.target\n" |
| 165 | ) |
| 166 | } |
| 167 | |
| 168 | /// Write a file inside the container by piping it to `tee`. |
| 169 | /// |
| 170 | /// `incus file push` would need a temp file on the host for content we generate |
| 171 | /// in memory. |
| 172 | fn write_inside(&self, instance: &str, path: &str, contents: &str) -> Result<()> { |
| 173 | use std::io::Write; |
| 174 | use std::process::Stdio; |
| 175 | |
| 176 | let mut child = std::process::Command::new("incus") |
| 177 | .args(["exec", instance, "--", "tee", path]) |
| 178 | .stdin(Stdio::piped()) |
| 179 | .stdout(Stdio::null()) |
| 180 | .stderr(Stdio::piped()) |
| 181 | .spawn() |
| 182 | .map_err(|e| Error::Internal(anyhow::Error::from(e).context("spawn incus exec")))?; |
| 183 | |
| 184 | child |
| 185 | .stdin |
| 186 | .take() |
| 187 | .expect("stdin is piped") |
| 188 | .write_all(contents.as_bytes()) |
| 189 | .map_err(|e| Error::Internal(anyhow::Error::from(e).context("write to container")))?; |
| 190 | |
| 191 | let output = child |
| 192 | .wait_with_output() |
| 193 | .map_err(|e| Error::Internal(anyhow::Error::from(e).context("wait for incus exec")))?; |
| 194 | |
| 195 | if output.status.success() { |
| 196 | Ok(()) |
| 197 | } else { |
| 198 | Err(Error::Internal(anyhow::anyhow!( |
| 199 | "writing {path} failed: {}", |
| 200 | String::from_utf8_lossy(&output.stderr).trim() |
| 201 | ))) |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | /// Wait for the container's network to come up. |
| 206 | /// |
| 207 | /// A freshly launched container has no address for a second or two, and the |
| 208 | /// endpoint is useless without one. Bounded, because a container that never |
| 209 | /// gets an address is a failure rather than something to wait on forever. |
| 210 | fn await_address(&self, account: &Name) -> Option<String> { |
| 211 | for _ in 0..30 { |
| 212 | if let Some(address) = self.address(account) { |
| 213 | return Some(address); |
| 214 | } |
| 215 | std::thread::sleep(std::time::Duration::from_millis(500)); |
| 216 | } |
| 217 | None |
| 218 | } |
| 219 | |
| 220 | /// Argument vectors, exposed so they can be asserted without running Incus. |
| 221 | /// |
| 222 | /// This is where the untestable part is made testable: the commands are pure |
| 223 | /// data, so a wrong flag is caught here rather than on a Linux host. |
| 224 | pub fn launch_args(&self, account: &Name) -> Vec<String> { |
| 225 | let name = instance_name(account); |
| 226 | vec![ |
| 227 | "launch".into(), |
| 228 | self.image.clone(), |
| 229 | name, |
| 230 | "--network".into(), |
| 231 | self.network.clone(), |
| 232 | // A tenant runs the account's own CI. Unprivileged is the default and is |
| 233 | // stated explicitly so a future edit cannot quietly drop it. |
| 234 | "-c".into(), |
| 235 | "security.privileged=false".into(), |
| 236 | "-c".into(), |
| 237 | "security.nesting=false".into(), |
| 238 | ] |
| 239 | } |
| 240 | |
| 241 | pub fn address_args(&self, account: &Name) -> Vec<String> { |
| 242 | vec![ |
| 243 | "list".into(), |
| 244 | instance_name(account), |
| 245 | "--format".into(), |
| 246 | "csv".into(), |
| 247 | "-c".into(), |
| 248 | "4".into(), |
| 249 | ] |
| 250 | } |
| 251 | |
| 252 | fn run(&self, args: &[String]) -> Result<String> { |
| 253 | let output = std::process::Command::new("incus") |
| 254 | .args(args) |
| 255 | .output() |
| 256 | .map_err(|e| { |
| 257 | Error::Internal(anyhow::Error::from(e).context("spawn incus; is it installed?")) |
| 258 | })?; |
| 259 | |
| 260 | if output.status.success() { |
| 261 | Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) |
| 262 | } else { |
| 263 | Err(Error::Internal(anyhow::anyhow!( |
| 264 | "incus {} failed: {}", |
| 265 | args.first().cloned().unwrap_or_default(), |
| 266 | String::from_utf8_lossy(&output.stderr).trim() |
| 267 | ))) |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | /// Whether an instance exists, whatever its state. |
| 272 | fn exists(&self, account: &Name) -> bool { |
| 273 | std::process::Command::new("incus") |
| 274 | .args(["info", &instance_name(account)]) |
| 275 | .output() |
| 276 | .map(|o| o.status.success()) |
| 277 | .unwrap_or(false) |
| 278 | } |
| 279 | |
| 280 | /// First IPv4 address Incus reports for the instance. |
| 281 | fn address(&self, account: &Name) -> Option<String> { |
| 282 | let out = self.run(&self.address_args(account)).ok()?; |
| 283 | out.lines() |
| 284 | .flat_map(|line| line.split_whitespace()) |
| 285 | .find(|token| token.contains('.') && token.parse::<std::net::Ipv4Addr>().is_ok()) |
| 286 | .map(String::from) |
| 287 | } |
| 288 | |
| 289 | fn endpoint(&self, account: &Name) -> Option<String> { |
| 290 | self.address(account) |
| 291 | .map(|ip| format!("http://{ip}:{}", self.port)) |
| 292 | } |
| 293 | } |
| 294 | |
| 295 | impl Provisioner for IncusProvisioner { |
| 296 | fn create(&self, account: &Name) -> Result<Instance> { |
| 297 | let instance = instance_name(account); |
| 298 | let product = crate::brand::NAME; |
| 299 | |
| 300 | if !self.exists(account) { |
| 301 | self.run(&self.launch_args(account))?; |
| 302 | } else { |
| 303 | // Idempotent: the reconciler retries, and a half-created instance must |
| 304 | // be driven forward rather than duplicated. |
| 305 | let _ = self.run(&["start".into(), instance.clone()]); |
| 306 | } |
| 307 | |
| 308 | // An address is needed before the tenant is worth starting, and before the |
| 309 | // endpoint means anything. |
| 310 | let address = self |
| 311 | .await_address(account) |
| 312 | .ok_or_else(|| Error::Internal(anyhow::anyhow!("{instance} never got an address")))?; |
| 313 | |
| 314 | // `--mode 0755`: a pushed file otherwise arrives without the execute bit. |
| 315 | self.run(&[ |
| 316 | "file".into(), |
| 317 | "push".into(), |
| 318 | "--quiet".into(), |
| 319 | self.binary.clone(), |
| 320 | format!("{instance}/usr/local/bin/{product}"), |
| 321 | "--mode".into(), |
| 322 | "0755".into(), |
| 323 | ])?; |
| 324 | |
| 325 | self.write_inside( |
| 326 | &instance, |
| 327 | &format!("/etc/{product}.env"), |
| 328 | &self.tenant_env(), |
| 329 | )?; |
| 330 | self.write_inside( |
| 331 | &instance, |
| 332 | &format!("/etc/systemd/system/{product}.service"), |
| 333 | &self.tenant_unit(), |
| 334 | )?; |
| 335 | |
| 336 | // git is what the tenant actually serves; the image does not ship it. |
| 337 | self.run(&[ |
| 338 | "exec".into(), |
| 339 | instance.clone(), |
| 340 | "--".into(), |
| 341 | "sh".into(), |
| 342 | "-c".into(), |
| 343 | "command -v git >/dev/null || (apt-get update -qq && apt-get install -y -qq git)" |
| 344 | .into(), |
| 345 | ])?; |
| 346 | |
| 347 | self.run(&[ |
| 348 | "exec".into(), |
| 349 | instance.clone(), |
| 350 | "--".into(), |
| 351 | "systemctl".into(), |
| 352 | "enable".into(), |
| 353 | "--now".into(), |
| 354 | product.into(), |
| 355 | ])?; |
| 356 | |
| 357 | Ok(Instance { |
| 358 | endpoint: format!("http://{address}:{}", self.port), |
| 359 | state: State::Provisioning, |
| 360 | }) |
| 361 | } |
| 362 | |
| 363 | fn upgrade(&self, account: &Name) -> Result<()> { |
| 364 | let instance = instance_name(account); |
| 365 | let product = crate::brand::NAME; |
| 366 | |
| 367 | if !self.exists(account) { |
| 368 | return Err(Error::NotFound("instance")); |
| 369 | } |
| 370 | |
| 371 | // Push beside the target, then rename over it. The running tenant keeps |
| 372 | // the file it is executing open, and Linux refuses to open a running |
| 373 | // executable for writing (ETXTBSY) — a direct push to the real path fails |
| 374 | // with "text file busy" on every running tenant, forever. rename(2) |
| 375 | // replaces the name while the old inode lives on under the process; the |
| 376 | // restart below is what moves the service onto the new build. The same |
| 377 | // dance the host deploy performs, for the same reason. |
| 378 | let staged = format!("/usr/local/bin/{product}.next"); |
| 379 | self.run(&[ |
| 380 | "file".into(), |
| 381 | "push".into(), |
| 382 | "--quiet".into(), |
| 383 | self.binary.clone(), |
| 384 | format!("{instance}{staged}"), |
| 385 | "--mode".into(), |
| 386 | "0755".into(), |
| 387 | ])?; |
| 388 | self.run(&[ |
| 389 | "exec".into(), |
| 390 | instance.clone(), |
| 391 | "--".into(), |
| 392 | "mv".into(), |
| 393 | "-f".into(), |
| 394 | staged, |
| 395 | format!("/usr/local/bin/{product}"), |
| 396 | ])?; |
| 397 | |
| 398 | // The env is rewritten too: a new version may read settings the old one did |
| 399 | // not, and a tenant that is upgraded but not reconfigured is a subtle |
| 400 | // version skew rather than a loud failure. |
| 401 | self.write_inside( |
| 402 | &instance, |
| 403 | &format!("/etc/{product}.env"), |
| 404 | &self.tenant_env(), |
| 405 | )?; |
| 406 | |
| 407 | self.run(&[ |
| 408 | "exec".into(), |
| 409 | instance.clone(), |
| 410 | "--".into(), |
| 411 | "systemctl".into(), |
| 412 | "restart".into(), |
| 413 | product.into(), |
| 414 | ])?; |
| 415 | |
| 416 | // Prove it came back. A tenant that fails to start after an upgrade must be |
| 417 | // reported, not recorded as upgraded. |
| 418 | for _ in 0..20 { |
| 419 | std::thread::sleep(std::time::Duration::from_millis(500)); |
| 420 | let active = std::process::Command::new("incus") |
| 421 | .args(["exec", &instance, "--", "systemctl", "is-active", product]) |
| 422 | .output() |
| 423 | .map(|o| String::from_utf8_lossy(&o.stdout).trim() == "active") |
| 424 | .unwrap_or(false); |
| 425 | if active { |
| 426 | return Ok(()); |
| 427 | } |
| 428 | } |
| 429 | Err(Error::Internal(anyhow::anyhow!( |
| 430 | "{instance} did not come back after upgrade" |
| 431 | ))) |
| 432 | } |
| 433 | |
| 434 | fn delete(&self, account: &Name) -> Result<()> { |
| 435 | if !self.exists(account) { |
| 436 | return Ok(()); |
| 437 | } |
| 438 | self.run(&["delete".into(), instance_name(account), "--force".into()]) |
| 439 | .map(|_| ()) |
| 440 | } |
| 441 | |
| 442 | fn status(&self, account: &Name) -> Result<Option<Instance>> { |
| 443 | if !self.exists(account) { |
| 444 | return Ok(None); |
| 445 | } |
| 446 | let running = self |
| 447 | .run(&[ |
| 448 | "list".into(), |
| 449 | instance_name(account), |
| 450 | "--format".into(), |
| 451 | "csv".into(), |
| 452 | "-c".into(), |
| 453 | "s".into(), |
| 454 | ]) |
| 455 | .map(|s| s.to_ascii_lowercase().contains("running")) |
| 456 | .unwrap_or(false); |
| 457 | |
| 458 | Ok(Some(Instance { |
| 459 | endpoint: self.endpoint(account).unwrap_or_default(), |
| 460 | state: if running { |
| 461 | State::Ready |
| 462 | } else { |
| 463 | State::Stopped |
| 464 | }, |
| 465 | })) |
| 466 | } |
| 467 | |
| 468 | fn start(&self, account: &Name) -> Result<Instance> { |
| 469 | self.run(&["start".into(), instance_name(account)])?; |
| 470 | Ok(Instance { |
| 471 | endpoint: self.endpoint(account).unwrap_or_default(), |
| 472 | state: State::Ready, |
| 473 | }) |
| 474 | } |
| 475 | |
| 476 | fn stop(&self, account: &Name) -> Result<()> { |
| 477 | self.run(&["stop".into(), instance_name(account)]) |
| 478 | .map(|_| ()) |
| 479 | } |
| 480 | } |
| 481 | |
| 482 | // ── Fake ─────────────────────────────────────────────────────────────────── |
| 483 | |
| 484 | /// An in-memory provisioner for tests. |
| 485 | /// |
| 486 | /// Everything above the Incus shim is exercised against this, which is what makes |
| 487 | /// the untestable part small rather than the untested part large. |
| 488 | /// |
| 489 | /// Also carries a `fail_next` switch, because the interesting paths are the failing |
| 490 | /// ones: a reconciler that only works when provisioning succeeds is not a reconciler. |
| 491 | #[cfg(test)] |
| 492 | #[derive(Default)] |
| 493 | pub struct FakeProvisioner { |
| 494 | instances: std::sync::Mutex<std::collections::HashMap<String, Instance>>, |
| 495 | pub fail_next: std::sync::atomic::AtomicBool, |
| 496 | /// Accounts upgraded, in order, so tests can assert what happened. |
| 497 | pub upgrades: std::sync::Mutex<Vec<String>>, |
| 498 | } |
| 499 | |
| 500 | #[cfg(test)] |
| 501 | impl FakeProvisioner { |
| 502 | pub fn new() -> Self { |
| 503 | Self::default() |
| 504 | } |
| 505 | |
| 506 | fn fail_if_asked(&self) -> Result<()> { |
| 507 | use std::sync::atomic::Ordering; |
| 508 | if self.fail_next.swap(false, Ordering::SeqCst) { |
| 509 | return Err(Error::Internal(anyhow::anyhow!("provisioning failed"))); |
| 510 | } |
| 511 | Ok(()) |
| 512 | } |
| 513 | } |
| 514 | |
| 515 | #[cfg(test)] |
| 516 | impl Provisioner for FakeProvisioner { |
| 517 | fn create(&self, account: &Name) -> Result<Instance> { |
| 518 | self.fail_if_asked()?; |
| 519 | let instance = Instance { |
| 520 | endpoint: format!("http://127.0.0.1:9/{}", account.as_str()), |
| 521 | state: State::Provisioning, |
| 522 | }; |
| 523 | self.instances |
| 524 | .lock() |
| 525 | .expect("fake lock") |
| 526 | .insert(account.as_str().to_string(), instance.clone()); |
| 527 | Ok(instance) |
| 528 | } |
| 529 | |
| 530 | fn upgrade(&self, account: &Name) -> Result<()> { |
| 531 | self.fail_if_asked()?; |
| 532 | self.upgrades |
| 533 | .lock() |
| 534 | .expect("fake lock") |
| 535 | .push(account.as_str().to_string()); |
| 536 | Ok(()) |
| 537 | } |
| 538 | |
| 539 | fn delete(&self, account: &Name) -> Result<()> { |
| 540 | self.instances |
| 541 | .lock() |
| 542 | .expect("fake lock") |
| 543 | .remove(account.as_str()); |
| 544 | Ok(()) |
| 545 | } |
| 546 | |
| 547 | fn status(&self, account: &Name) -> Result<Option<Instance>> { |
| 548 | Ok(self |
| 549 | .instances |
| 550 | .lock() |
| 551 | .expect("fake lock") |
| 552 | .get(account.as_str()) |
| 553 | .cloned() |
| 554 | .map(|mut i| { |
| 555 | // A fake container becomes ready the moment it is asked about. |
| 556 | if i.state == State::Provisioning { |
| 557 | i.state = State::Ready; |
| 558 | } |
| 559 | i |
| 560 | })) |
| 561 | } |
| 562 | |
| 563 | fn start(&self, account: &Name) -> Result<Instance> { |
| 564 | let mut guard = self.instances.lock().expect("fake lock"); |
| 565 | let instance = guard |
| 566 | .get_mut(account.as_str()) |
| 567 | .ok_or(Error::NotFound("instance"))?; |
| 568 | instance.state = State::Ready; |
| 569 | Ok(instance.clone()) |
| 570 | } |
| 571 | |
| 572 | fn stop(&self, account: &Name) -> Result<()> { |
| 573 | let mut guard = self.instances.lock().expect("fake lock"); |
| 574 | if let Some(instance) = guard.get_mut(account.as_str()) { |
| 575 | instance.state = State::Stopped; |
| 576 | } |
| 577 | Ok(()) |
| 578 | } |
| 579 | } |
| 580 | |
| 581 | #[cfg(test)] |
| 582 | mod tests { |
| 583 | use super::*; |
| 584 | use crate::git::validate::name; |
| 585 | |
| 586 | fn incus() -> IncusProvisioner { |
| 587 | IncusProvisioner::new( |
| 588 | "images:ubuntu/24.04".into(), |
| 589 | "testbr0".into(), |
| 590 | 8790, |
| 591 | format!("/usr/local/bin/{}", crate::brand::NAME), |
| 592 | "dGVzdA==".into(), |
| 593 | PublicAddresses { |
| 594 | url: Some("https://git.example.com".into()), |
| 595 | ssh: Some("ssh://git@git.example.com:2222".into()), |
| 596 | package: Some("https://pkg.git.example.com".into()), |
| 597 | }, |
| 598 | ) |
| 599 | } |
| 600 | |
| 601 | #[test] |
| 602 | fn an_instance_name_is_derived_from_the_account_and_the_product() { |
| 603 | let derived = instance_name(&name("alice").unwrap()); |
| 604 | assert!(derived.contains("alice")); |
| 605 | assert!( |
| 606 | derived.starts_with(crate::brand::NAME), |
| 607 | "the prefix must track the product name, not be a literal" |
| 608 | ); |
| 609 | } |
| 610 | |
| 611 | #[test] |
| 612 | fn launch_pins_the_isolation_settings_that_matter() { |
| 613 | // These are the arguments that decide whether a tenant is contained. They |
| 614 | // are asserted here because this is the one part of the control plane that |
| 615 | // cannot be exercised without a Linux host. |
| 616 | let args = incus().launch_args(&name("alice").unwrap()).join(" "); |
| 617 | |
| 618 | assert!(args.contains("security.privileged=false"), "{args}"); |
| 619 | assert!(args.contains("security.nesting=false"), "{args}"); |
| 620 | assert!(args.contains("--network testbr0"), "{args}"); |
| 621 | assert!(args.starts_with("launch images:ubuntu/24.04"), "{args}"); |
| 622 | } |
| 623 | |
| 624 | #[test] |
| 625 | fn the_tenant_is_told_which_key_may_vouch_for_a_caller() { |
| 626 | let env = incus().tenant_env(); |
| 627 | assert!(env.contains("MODE=tenant")); |
| 628 | assert!( |
| 629 | env.contains("CONTROL_PUBLIC_KEYS=dGVzdA=="), |
| 630 | "a tenant that trusts no key cannot be reached: {env}" |
| 631 | ); |
| 632 | assert!( |
| 633 | env.contains("SSH_BIND=off"), |
| 634 | "SSH terminates at the control plane" |
| 635 | ); |
| 636 | } |
| 637 | |
| 638 | #[test] |
| 639 | fn a_tenant_advertises_the_public_address_not_its_own() { |
| 640 | // A tenant sits on a private bridge and is only reachable through the |
| 641 | // control plane, so left to itself it hands callers a clone URL for a |
| 642 | // container they cannot reach. This was live on real infrastructure until a |
| 643 | // deployment surfaced it: `"clone_url_http":"http://0.0.0.0:8790/..."`. |
| 644 | let env = incus().tenant_env(); |
| 645 | assert!(env.contains("PUBLIC_URL=https://git.example.com"), "{env}"); |
| 646 | assert!( |
| 647 | env.contains("PUBLIC_SSH=ssh://git@git.example.com:2222"), |
| 648 | "{env}" |
| 649 | ); |
| 650 | assert!( |
| 651 | env.contains("PACKAGE_URL=https://pkg.git.example.com"), |
| 652 | "{env}" |
| 653 | ); |
| 654 | } |
| 655 | |
| 656 | #[test] |
| 657 | fn an_unconfigured_public_address_is_omitted_rather_than_guessed() { |
| 658 | // Without a domain there is no correct answer, and a guessed one is worse |
| 659 | // than none: the tenant falls back to its bind address, which is at least |
| 660 | // obviously wrong rather than subtly wrong. |
| 661 | let bare = IncusProvisioner::new( |
| 662 | "images:debian/12".into(), |
| 663 | "incusbr0".into(), |
| 664 | 8790, |
| 665 | "/usr/local/bin/x".into(), |
| 666 | "dGVzdA==".into(), |
| 667 | PublicAddresses::default(), |
| 668 | ); |
| 669 | assert!(!bare.tenant_env().contains("PUBLIC_URL")); |
| 670 | } |
| 671 | |
| 672 | #[test] |
| 673 | fn the_tenant_unit_restarts_and_reads_its_environment() { |
| 674 | let unit = incus().tenant_unit(); |
| 675 | assert!(unit.contains("Restart=always")); |
| 676 | assert!(unit.contains(&format!("EnvironmentFile=/etc/{}.env", crate::brand::NAME))); |
| 677 | } |
| 678 | |
| 679 | #[test] |
| 680 | fn a_fake_tenant_moves_through_its_lifecycle() { |
| 681 | let fake = FakeProvisioner::new(); |
| 682 | let alice = name("alice").unwrap(); |
| 683 | |
| 684 | assert!(fake.status(&alice).unwrap().is_none()); |
| 685 | |
| 686 | let created = fake.create(&alice).unwrap(); |
| 687 | assert_eq!(created.state, State::Provisioning); |
| 688 | assert_eq!(fake.status(&alice).unwrap().unwrap().state, State::Ready); |
| 689 | |
| 690 | fake.stop(&alice).unwrap(); |
| 691 | assert_eq!(fake.status(&alice).unwrap().unwrap().state, State::Stopped); |
| 692 | |
| 693 | fake.start(&alice).unwrap(); |
| 694 | assert_eq!(fake.status(&alice).unwrap().unwrap().state, State::Ready); |
| 695 | |
| 696 | fake.delete(&alice).unwrap(); |
| 697 | assert!(fake.status(&alice).unwrap().is_none()); |
| 698 | } |
| 699 | |
| 700 | #[test] |
| 701 | fn deleting_an_absent_tenant_is_not_an_error() { |
| 702 | // The reconciler and the delete path both retry; neither should fail on a |
| 703 | // tenant that is already gone. |
| 704 | let fake = FakeProvisioner::new(); |
| 705 | fake.delete(&name("ghost").unwrap()).unwrap(); |
| 706 | } |
| 707 | |
| 708 | #[test] |
| 709 | fn a_provisioning_failure_surfaces_rather_than_being_swallowed() { |
| 710 | use std::sync::atomic::Ordering; |
| 711 | let fake = FakeProvisioner::new(); |
| 712 | fake.fail_next.store(true, Ordering::SeqCst); |
| 713 | |
| 714 | assert!(fake.create(&name("alice").unwrap()).is_err()); |
| 715 | // And the switch is one-shot, so the retry succeeds. |
| 716 | fake.create(&name("alice").unwrap()).unwrap(); |
| 717 | } |
| 718 | } |