zuka
zuka/src/control/provision.rs

worklyn / zukapublic

Agent-first git hosting. One Rust binary: git over HTTP and SSH, a REST API, MCP, CI, and multi-tenant isolation.

Get a copy: git clone https://zuka.worklyn.com/worklyn/zuka.git
branchbc985d6bbd321be340e3750b3da4734ee1553cbd ▾
main All branches →
zuka/src/control/provision.rs
RSprovision.rs23.5 KBDownload
1// Provisioning a tenant.
2//
3// The container operations sit behind a trait for one reason: they are the only part
4// of the control plane that cannot be tested without a Linux host running Incus.
5// Everything above this line — the state machine, the reconciler, routing, the
6// signed assertion — is exercised against `FakeProvisioner` on any machine, and the
7// Incus implementation stays a thin, reviewable shim over the CLI.
8//
9// The methods are synchronous. Each is a subprocess spawn, so they run through
10// `git::exec::blocking` at the call site, matching how the rest of the service
11// treats blocking work.
12
13use crate::error::{Error, Result};
14use crate::git::validate::Name;
15use serde::{Deserialize, Serialize};
16
17/// Where a tenant is in its lifecycle.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "snake_case")]
20pub enum State {
21 /// Requested. The reconciler will drive it forward.
22 Provisioning,
23 /// Running and reachable.
24 Ready,
25 /// Stopped to save resources; started again on demand.
26 Stopped,
27 /// Provisioning failed. `detail` says why.
28 Failed,
29}
30
31/// A provisioned tenant.
32#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
33pub struct Instance {
34 /// Address the control plane proxies to.
35 pub endpoint: String,
36 pub state: State,
37}
38
39pub trait Provisioner: Send + Sync {
40 /// Create and start a tenant. Must be idempotent: the reconciler retries.
41 fn create(&self, account: &Name) -> Result<Instance>;
42 fn delete(&self, account: &Name) -> Result<()>;
43 fn status(&self, account: &Name) -> Result<Option<Instance>>;
44 /// Replace the binary in a running tenant and restart it.
45 ///
46 /// Separate from `create` because it must not relaunch the container: the
47 /// tenant's repositories live on its own disk, and recreating it to change a
48 /// binary would destroy them.
49 fn upgrade(&self, account: &Name) -> Result<()>;
50 fn start(&self, account: &Name) -> Result<Instance>;
51 fn stop(&self, account: &Name) -> Result<()>;
52}
53
54/// Instance name for an account. Incus names are DNS-ish, and our account names are
55/// already validated to a stricter set, so the prefix is the only transformation.
56pub fn instance_name(account: &Name) -> String {
57 format!("{}-{}", crate::brand::NAME, account.as_str())
58}
59
60// ── Incus ──────────────────────────────────────────────────────────────────
61
62/// Drives Incus through its CLI.
63///
64/// The CLI rather than the REST socket: it is the documented interface, it handles
65/// authentication against a remote host, and it keeps this shim small enough to read
66/// in one sitting — which matters because it is the part with the least test
67/// coverage.
68pub struct IncusProvisioner {
69 /// Image to launch, e.g. `images:debian/12`.
70 image: String,
71 /// Network the container joins.
72 network: String,
73 /// Port the tenant listens on inside its container.
74 port: u16,
75 /// Where the tenant binary lives on the host, to push into the container.
76 ///
77 /// Must be statically linked: the host and the container image do not share a
78 /// C library, and a dynamically linked binary fails at exec inside the
79 /// container with a glibc version error.
80 binary: String,
81 /// Public half of the control plane's signing key, so the tenant can verify
82 /// the assertions it is sent. Public by construction — it forges nothing.
83 control_public_key: String,
84 /// What the outside world calls this service.
85 ///
86 /// A tenant is only ever reached through the control plane, so it cannot know
87 /// its own public address — left to itself it advertises its bind address, and
88 /// a user is handed `http://0.0.0.0:8790/...` as a clone URL.
89 public_url: Option<String>,
90 public_ssh: Option<String>,
91}
92
93impl IncusProvisioner {
94 pub fn new(
95 image: String,
96 network: String,
97 port: u16,
98 binary: String,
99 control_public_key: String,
100 public_url: Option<String>,
101 public_ssh: Option<String>,
102 ) -> Self {
103 IncusProvisioner {
104 image,
105 network,
106 port,
107 binary,
108 control_public_key,
109 public_url,
110 public_ssh,
111 }
112 }
113
114 /// The tenant's environment file, written inside the container.
115 fn tenant_env(&self) -> String {
116 let p = crate::brand::env_prefix();
117 let mut lines = vec![
118 format!("{p}MODE=tenant"),
119 format!("{p}DATA_DIR=/var/lib/{}", crate::brand::NAME),
120 format!("{p}BIND=0.0.0.0:{}", self.port),
121 // SSH terminates at the control plane; a tenant never listens for it.
122 format!("{p}SSH_BIND=off"),
123 format!("{p}CONTROL_PUBLIC_KEYS={}", self.control_public_key),
124 format!("{p}CI_ENABLED=1"),
125 ];
126
127 // Without these the tenant advertises its own bind address, and a caller is
128 // handed a clone URL pointing at a container they cannot reach.
129 if let Some(url) = &self.public_url {
130 lines.push(format!("{p}PUBLIC_URL={url}"));
131 }
132 if let Some(ssh) = &self.public_ssh {
133 lines.push(format!("{p}PUBLIC_SSH={ssh}"));
134 }
135 lines.join("\n") + "\n"
136 }
137
138 /// The systemd unit that runs the tenant inside its container.
139 fn tenant_unit(&self) -> String {
140 let name = crate::brand::NAME;
141 format!(
142 "[Unit]\n\
143 Description={name} tenant\n\
144 After=network-online.target\n\
145 \n\
146 [Service]\n\
147 ExecStart=/usr/local/bin/{name}\n\
148 EnvironmentFile=/etc/{name}.env\n\
149 Restart=always\n\
150 RestartSec=2\n\
151 \n\
152 [Install]\n\
153 WantedBy=multi-user.target\n"
154 )
155 }
156
157 /// Write a file inside the container by piping it to `tee`.
158 ///
159 /// `incus file push` would need a temp file on the host for content we generate
160 /// in memory.
161 fn write_inside(&self, instance: &str, path: &str, contents: &str) -> Result<()> {
162 use std::io::Write;
163 use std::process::Stdio;
164
165 let mut child = std::process::Command::new("incus")
166 .args(["exec", instance, "--", "tee", path])
167 .stdin(Stdio::piped())
168 .stdout(Stdio::null())
169 .stderr(Stdio::piped())
170 .spawn()
171 .map_err(|e| Error::Internal(anyhow::Error::from(e).context("spawn incus exec")))?;
172
173 child
174 .stdin
175 .take()
176 .expect("stdin is piped")
177 .write_all(contents.as_bytes())
178 .map_err(|e| Error::Internal(anyhow::Error::from(e).context("write to container")))?;
179
180 let output = child
181 .wait_with_output()
182 .map_err(|e| Error::Internal(anyhow::Error::from(e).context("wait for incus exec")))?;
183
184 if output.status.success() {
185 Ok(())
186 } else {
187 Err(Error::Internal(anyhow::anyhow!(
188 "writing {path} failed: {}",
189 String::from_utf8_lossy(&output.stderr).trim()
190 )))
191 }
192 }
193
194 /// Wait for the container's network to come up.
195 ///
196 /// A freshly launched container has no address for a second or two, and the
197 /// endpoint is useless without one. Bounded, because a container that never
198 /// gets an address is a failure rather than something to wait on forever.
199 fn await_address(&self, account: &Name) -> Option<String> {
200 for _ in 0..30 {
201 if let Some(address) = self.address(account) {
202 return Some(address);
203 }
204 std::thread::sleep(std::time::Duration::from_millis(500));
205 }
206 None
207 }
208
209 /// Argument vectors, exposed so they can be asserted without running Incus.
210 ///
211 /// This is where the untestable part is made testable: the commands are pure
212 /// data, so a wrong flag is caught here rather than on a Linux host.
213 pub fn launch_args(&self, account: &Name) -> Vec<String> {
214 let name = instance_name(account);
215 vec![
216 "launch".into(),
217 self.image.clone(),
218 name,
219 "--network".into(),
220 self.network.clone(),
221 // A tenant runs the account's own CI. Unprivileged is the default and is
222 // stated explicitly so a future edit cannot quietly drop it.
223 "-c".into(),
224 "security.privileged=false".into(),
225 "-c".into(),
226 "security.nesting=false".into(),
227 ]
228 }
229
230 pub fn address_args(&self, account: &Name) -> Vec<String> {
231 vec![
232 "list".into(),
233 instance_name(account),
234 "--format".into(),
235 "csv".into(),
236 "-c".into(),
237 "4".into(),
238 ]
239 }
240
241 fn run(&self, args: &[String]) -> Result<String> {
242 let output = std::process::Command::new("incus")
243 .args(args)
244 .output()
245 .map_err(|e| {
246 Error::Internal(anyhow::Error::from(e).context("spawn incus; is it installed?"))
247 })?;
248
249 if output.status.success() {
250 Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
251 } else {
252 Err(Error::Internal(anyhow::anyhow!(
253 "incus {} failed: {}",
254 args.first().cloned().unwrap_or_default(),
255 String::from_utf8_lossy(&output.stderr).trim()
256 )))
257 }
258 }
259
260 /// Whether an instance exists, whatever its state.
261 fn exists(&self, account: &Name) -> bool {
262 std::process::Command::new("incus")
263 .args(["info", &instance_name(account)])
264 .output()
265 .map(|o| o.status.success())
266 .unwrap_or(false)
267 }
268
269 /// First IPv4 address Incus reports for the instance.
270 fn address(&self, account: &Name) -> Option<String> {
271 let out = self.run(&self.address_args(account)).ok()?;
272 out.lines()
273 .flat_map(|line| line.split_whitespace())
274 .find(|token| token.contains('.') && token.parse::<std::net::Ipv4Addr>().is_ok())
275 .map(String::from)
276 }
277
278 fn endpoint(&self, account: &Name) -> Option<String> {
279 self.address(account)
280 .map(|ip| format!("http://{ip}:{}", self.port))
281 }
282}
283
284impl Provisioner for IncusProvisioner {
285 fn create(&self, account: &Name) -> Result<Instance> {
286 let instance = instance_name(account);
287 let product = crate::brand::NAME;
288
289 if !self.exists(account) {
290 self.run(&self.launch_args(account))?;
291 } else {
292 // Idempotent: the reconciler retries, and a half-created instance must
293 // be driven forward rather than duplicated.
294 let _ = self.run(&["start".into(), instance.clone()]);
295 }
296
297 // An address is needed before the tenant is worth starting, and before the
298 // endpoint means anything.
299 let address = self
300 .await_address(account)
301 .ok_or_else(|| Error::Internal(anyhow::anyhow!("{instance} never got an address")))?;
302
303 // `--mode 0755`: a pushed file otherwise arrives without the execute bit.
304 self.run(&[
305 "file".into(),
306 "push".into(),
307 "--quiet".into(),
308 self.binary.clone(),
309 format!("{instance}/usr/local/bin/{product}"),
310 "--mode".into(),
311 "0755".into(),
312 ])?;
313
314 self.write_inside(
315 &instance,
316 &format!("/etc/{product}.env"),
317 &self.tenant_env(),
318 )?;
319 self.write_inside(
320 &instance,
321 &format!("/etc/systemd/system/{product}.service"),
322 &self.tenant_unit(),
323 )?;
324
325 // git is what the tenant actually serves; the image does not ship it.
326 self.run(&[
327 "exec".into(),
328 instance.clone(),
329 "--".into(),
330 "sh".into(),
331 "-c".into(),
332 "command -v git >/dev/null || (apt-get update -qq && apt-get install -y -qq git)"
333 .into(),
334 ])?;
335
336 self.run(&[
337 "exec".into(),
338 instance.clone(),
339 "--".into(),
340 "systemctl".into(),
341 "enable".into(),
342 "--now".into(),
343 product.into(),
344 ])?;
345
346 Ok(Instance {
347 endpoint: format!("http://{address}:{}", self.port),
348 state: State::Provisioning,
349 })
350 }
351
352 fn upgrade(&self, account: &Name) -> Result<()> {
353 let instance = instance_name(account);
354 let product = crate::brand::NAME;
355
356 if !self.exists(account) {
357 return Err(Error::NotFound("instance"));
358 }
359
360 // Push then restart. The container, its disk and its repositories are
361 // untouched; only the executable changes.
362 self.run(&[
363 "file".into(),
364 "push".into(),
365 "--quiet".into(),
366 self.binary.clone(),
367 format!("{instance}/usr/local/bin/{product}"),
368 "--mode".into(),
369 "0755".into(),
370 ])?;
371
372 // The env is rewritten too: a new version may read settings the old one did
373 // not, and a tenant that is upgraded but not reconfigured is a subtle
374 // version skew rather than a loud failure.
375 self.write_inside(
376 &instance,
377 &format!("/etc/{product}.env"),
378 &self.tenant_env(),
379 )?;
380
381 self.run(&[
382 "exec".into(),
383 instance.clone(),
384 "--".into(),
385 "systemctl".into(),
386 "restart".into(),
387 product.into(),
388 ])?;
389
390 // Prove it came back. A tenant that fails to start after an upgrade must be
391 // reported, not recorded as upgraded.
392 for _ in 0..20 {
393 std::thread::sleep(std::time::Duration::from_millis(500));
394 let active = std::process::Command::new("incus")
395 .args(["exec", &instance, "--", "systemctl", "is-active", product])
396 .output()
397 .map(|o| String::from_utf8_lossy(&o.stdout).trim() == "active")
398 .unwrap_or(false);
399 if active {
400 return Ok(());
401 }
402 }
403 Err(Error::Internal(anyhow::anyhow!(
404 "{instance} did not come back after upgrade"
405 )))
406 }
407
408 fn delete(&self, account: &Name) -> Result<()> {
409 if !self.exists(account) {
410 return Ok(());
411 }
412 self.run(&["delete".into(), instance_name(account), "--force".into()])
413 .map(|_| ())
414 }
415
416 fn status(&self, account: &Name) -> Result<Option<Instance>> {
417 if !self.exists(account) {
418 return Ok(None);
419 }
420 let running = self
421 .run(&[
422 "list".into(),
423 instance_name(account),
424 "--format".into(),
425 "csv".into(),
426 "-c".into(),
427 "s".into(),
428 ])
429 .map(|s| s.to_ascii_lowercase().contains("running"))
430 .unwrap_or(false);
431
432 Ok(Some(Instance {
433 endpoint: self.endpoint(account).unwrap_or_default(),
434 state: if running {
435 State::Ready
436 } else {
437 State::Stopped
438 },
439 }))
440 }
441
442 fn start(&self, account: &Name) -> Result<Instance> {
443 self.run(&["start".into(), instance_name(account)])?;
444 Ok(Instance {
445 endpoint: self.endpoint(account).unwrap_or_default(),
446 state: State::Ready,
447 })
448 }
449
450 fn stop(&self, account: &Name) -> Result<()> {
451 self.run(&["stop".into(), instance_name(account)])
452 .map(|_| ())
453 }
454}
455
456// ── Fake ───────────────────────────────────────────────────────────────────
457
458/// An in-memory provisioner for tests.
459///
460/// Everything above the Incus shim is exercised against this, which is what makes
461/// the untestable part small rather than the untested part large.
462///
463/// Also carries a `fail_next` switch, because the interesting paths are the failing
464/// ones: a reconciler that only works when provisioning succeeds is not a reconciler.
465#[cfg(test)]
466#[derive(Default)]
467pub struct FakeProvisioner {
468 instances: std::sync::Mutex<std::collections::HashMap<String, Instance>>,
469 pub fail_next: std::sync::atomic::AtomicBool,
470 /// Accounts upgraded, in order, so tests can assert what happened.
471 pub upgrades: std::sync::Mutex<Vec<String>>,
472}
473
474#[cfg(test)]
475impl FakeProvisioner {
476 pub fn new() -> Self {
477 Self::default()
478 }
479
480 fn fail_if_asked(&self) -> Result<()> {
481 use std::sync::atomic::Ordering;
482 if self.fail_next.swap(false, Ordering::SeqCst) {
483 return Err(Error::Internal(anyhow::anyhow!("provisioning failed")));
484 }
485 Ok(())
486 }
487}
488
489#[cfg(test)]
490impl Provisioner for FakeProvisioner {
491 fn create(&self, account: &Name) -> Result<Instance> {
492 self.fail_if_asked()?;
493 let instance = Instance {
494 endpoint: format!("http://127.0.0.1:9/{}", account.as_str()),
495 state: State::Provisioning,
496 };
497 self.instances
498 .lock()
499 .expect("fake lock")
500 .insert(account.as_str().to_string(), instance.clone());
501 Ok(instance)
502 }
503
504 fn upgrade(&self, account: &Name) -> Result<()> {
505 self.fail_if_asked()?;
506 self.upgrades
507 .lock()
508 .expect("fake lock")
509 .push(account.as_str().to_string());
510 Ok(())
511 }
512
513 fn delete(&self, account: &Name) -> Result<()> {
514 self.instances
515 .lock()
516 .expect("fake lock")
517 .remove(account.as_str());
518 Ok(())
519 }
520
521 fn status(&self, account: &Name) -> Result<Option<Instance>> {
522 Ok(self
523 .instances
524 .lock()
525 .expect("fake lock")
526 .get(account.as_str())
527 .cloned()
528 .map(|mut i| {
529 // A fake container becomes ready the moment it is asked about.
530 if i.state == State::Provisioning {
531 i.state = State::Ready;
532 }
533 i
534 }))
535 }
536
537 fn start(&self, account: &Name) -> Result<Instance> {
538 let mut guard = self.instances.lock().expect("fake lock");
539 let instance = guard
540 .get_mut(account.as_str())
541 .ok_or(Error::NotFound("instance"))?;
542 instance.state = State::Ready;
543 Ok(instance.clone())
544 }
545
546 fn stop(&self, account: &Name) -> Result<()> {
547 let mut guard = self.instances.lock().expect("fake lock");
548 if let Some(instance) = guard.get_mut(account.as_str()) {
549 instance.state = State::Stopped;
550 }
551 Ok(())
552 }
553}
554
555#[cfg(test)]
556mod tests {
557 use super::*;
558 use crate::git::validate::name;
559
560 fn incus() -> IncusProvisioner {
561 IncusProvisioner::new(
562 "images:ubuntu/24.04".into(),
563 "testbr0".into(),
564 8790,
565 format!("/usr/local/bin/{}", crate::brand::NAME),
566 "dGVzdA==".into(),
567 Some("https://git.example.com".into()),
568 Some("ssh://git@git.example.com:2222".into()),
569 )
570 }
571
572 #[test]
573 fn an_instance_name_is_derived_from_the_account_and_the_product() {
574 let derived = instance_name(&name("alice").unwrap());
575 assert!(derived.contains("alice"));
576 assert!(
577 derived.starts_with(crate::brand::NAME),
578 "the prefix must track the product name, not be a literal"
579 );
580 }
581
582 #[test]
583 fn launch_pins_the_isolation_settings_that_matter() {
584 // These are the arguments that decide whether a tenant is contained. They
585 // are asserted here because this is the one part of the control plane that
586 // cannot be exercised without a Linux host.
587 let args = incus().launch_args(&name("alice").unwrap()).join(" ");
588
589 assert!(args.contains("security.privileged=false"), "{args}");
590 assert!(args.contains("security.nesting=false"), "{args}");
591 assert!(args.contains("--network testbr0"), "{args}");
592 assert!(args.starts_with("launch images:ubuntu/24.04"), "{args}");
593 }
594
595 #[test]
596 fn the_tenant_is_told_which_key_may_vouch_for_a_caller() {
597 let env = incus().tenant_env();
598 assert!(env.contains("MODE=tenant"));
599 assert!(
600 env.contains("CONTROL_PUBLIC_KEYS=dGVzdA=="),
601 "a tenant that trusts no key cannot be reached: {env}"
602 );
603 assert!(
604 env.contains("SSH_BIND=off"),
605 "SSH terminates at the control plane"
606 );
607 }
608
609 #[test]
610 fn a_tenant_advertises_the_public_address_not_its_own() {
611 // A tenant sits on a private bridge and is only reachable through the
612 // control plane, so left to itself it hands callers a clone URL for a
613 // container they cannot reach. This was live on real infrastructure until a
614 // deployment surfaced it: `"clone_url_http":"http://0.0.0.0:8790/..."`.
615 let env = incus().tenant_env();
616 assert!(env.contains("PUBLIC_URL=https://git.example.com"), "{env}");
617 assert!(
618 env.contains("PUBLIC_SSH=ssh://git@git.example.com:2222"),
619 "{env}"
620 );
621 }
622
623 #[test]
624 fn an_unconfigured_public_address_is_omitted_rather_than_guessed() {
625 // Without a domain there is no correct answer, and a guessed one is worse
626 // than none: the tenant falls back to its bind address, which is at least
627 // obviously wrong rather than subtly wrong.
628 let bare = IncusProvisioner::new(
629 "images:debian/12".into(),
630 "incusbr0".into(),
631 8790,
632 "/usr/local/bin/x".into(),
633 "dGVzdA==".into(),
634 None,
635 None,
636 );
637 assert!(!bare.tenant_env().contains("PUBLIC_URL"));
638 }
639
640 #[test]
641 fn the_tenant_unit_restarts_and_reads_its_environment() {
642 let unit = incus().tenant_unit();
643 assert!(unit.contains("Restart=always"));
644 assert!(unit.contains(&format!("EnvironmentFile=/etc/{}.env", crate::brand::NAME)));
645 }
646
647 #[test]
648 fn a_fake_tenant_moves_through_its_lifecycle() {
649 let fake = FakeProvisioner::new();
650 let alice = name("alice").unwrap();
651
652 assert!(fake.status(&alice).unwrap().is_none());
653
654 let created = fake.create(&alice).unwrap();
655 assert_eq!(created.state, State::Provisioning);
656 assert_eq!(fake.status(&alice).unwrap().unwrap().state, State::Ready);
657
658 fake.stop(&alice).unwrap();
659 assert_eq!(fake.status(&alice).unwrap().unwrap().state, State::Stopped);
660
661 fake.start(&alice).unwrap();
662 assert_eq!(fake.status(&alice).unwrap().unwrap().state, State::Ready);
663
664 fake.delete(&alice).unwrap();
665 assert!(fake.status(&alice).unwrap().is_none());
666 }
667
668 #[test]
669 fn deleting_an_absent_tenant_is_not_an_error() {
670 // The reconciler and the delete path both retry; neither should fail on a
671 // tenant that is already gone.
672 let fake = FakeProvisioner::new();
673 fake.delete(&name("ghost").unwrap()).unwrap();
674 }
675
676 #[test]
677 fn a_provisioning_failure_surfaces_rather_than_being_swallowed() {
678 use std::sync::atomic::Ordering;
679 let fake = FakeProvisioner::new();
680 fake.fail_next.store(true, Ordering::SeqCst);
681
682 assert!(fake.create(&name("alice").unwrap()).is_err());
683 // And the switch is one-shot, so the retry succeeds.
684 fake.create(&name("alice").unwrap()).unwrap();
685 }
686}