zuka
zuka/src/jobs/gc.rs

worklyn / zukapublic

Agent-first git hosting. One Rust binary: git over HTTP and SSH, a REST API, MCP, CI, and multi-tenant isolation.

Get a copy: git clone https://zuka.worklyn.com/worklyn/zuka.git
branchbc985d6bbd321be340e3750b3da4734ee1553cbd ▾
main All branches →
zuka/src/jobs/gc.rs
RSgc.rs7.7 KBDownload
1// Garbage collection.
2//
3// Two things have to be true for this to be safe, and neither is automatic:
4//
5// * git's own in-push auto-gc must be OFF. It defaults ON via `receive.autogc`,
6// which means a repack fires inside `receive-pack` at a moment nobody chose,
7// slowing that push and running unsynchronised with everything else. Repos get
8// `gc.auto=0` at creation; this job reapplies it so repositories created before
9// the setting existed are corrected too.
10//
11// * pruning must keep a grace window. `upload-pack` can reference an object
12// between resolving it and streaming it, and pruning inside that window is the
13// classic way to hand a client a corrupt clone. The grace period is what makes
14// scheduled GC safe against a live clone — not a lock, which could not span the
15// two processes anyway.
16
17use crate::config::Config;
18use crate::git::exec::Git;
19use crate::jobs::walk_repos;
20use std::path::Path;
21
22#[derive(Debug, Default)]
23pub struct Outcome {
24 pub packed: usize,
25 pub reclaimed: u64,
26}
27
28pub fn run(config: &Config) -> Outcome {
29 let mut outcome = Outcome::default();
30
31 for (account, repo, path) in walk_repos(&config.data_dir.join("git")) {
32 let before = crate::store::quota::repo_bytes(&path);
33
34 if let Err(e) = collect(&path, &config.gc_prune_grace) {
35 eprintln!("[jobs] gc {account}/{repo} failed: {e}");
36 continue;
37 }
38
39 let after = crate::store::quota::repo_bytes(&path);
40 outcome.packed += 1;
41 outcome.reclaimed = outcome
42 .reclaimed
43 .saturating_add(before.saturating_sub(after));
44 }
45 outcome
46}
47
48/// Repack and prune one repository.
49pub fn collect(git_dir: &Path, prune_grace: &str) -> crate::error::Result<()> {
50 let git = Git::at(git_dir);
51
52 // Turn off in-push auto-gc before doing anything else, so a repository created
53 // by an older build stops repacking inside its own pushes.
54 git.ignore(&["config", "gc.auto", "0"]);
55 git.ignore(&["config", "receive.autogc", "false"]);
56
57 // `-A` keeps unreachable objects as loose so `prune` can apply the grace window
58 // to them; `-d` removes the packs made redundant by the new one.
59 git.run(&["repack", "-Ad", "--quiet"])?;
60 git.run(&["prune", &format!("--expire={prune_grace}")])?;
61 // Rewrite packed-refs, which nothing else ever compacts.
62 git.run(&["pack-refs", "--all", "--prune"])?;
63 Ok(())
64}
65
66/// Whether a repository is intact. Used by `fsck` and by tests.
67pub fn verify(git_dir: &Path) -> crate::error::Result<()> {
68 Git::at(git_dir)
69 .run(&["fsck", "--no-progress", "--connectivity-only"])
70 .map(|_| ())
71}
72
73#[cfg(test)]
74mod tests {
75 use super::*;
76 use std::io::Write;
77 use std::process::Command;
78 use std::process::Stdio;
79
80 fn bare() -> (tempfile::TempDir, std::path::PathBuf) {
81 let dir = tempfile::tempdir().unwrap();
82 let git_dir = dir.path().join("r.git");
83 Git::plain()
84 .run(&["init", "--bare", "--quiet", git_dir.to_str().unwrap()])
85 .unwrap();
86 (dir, git_dir)
87 }
88
89 /// Write a loose object and return its id.
90 fn loose(git_dir: &Path, content: &[u8]) -> String {
91 let mut child = Command::new("git")
92 .arg("--git-dir")
93 .arg(git_dir)
94 .args(["hash-object", "-w", "--stdin"])
95 .stdin(Stdio::piped())
96 .stdout(Stdio::piped())
97 .spawn()
98 .unwrap();
99 child.stdin.take().unwrap().write_all(content).unwrap();
100 let out = child.wait_with_output().unwrap();
101 String::from_utf8_lossy(&out.stdout).trim().to_string()
102 }
103
104 fn count_loose(git_dir: &Path) -> usize {
105 walkdir(&git_dir.join("objects"))
106 .into_iter()
107 .filter(|p| !p.to_string_lossy().contains("/pack/"))
108 .filter(|p| p.is_file())
109 .count()
110 }
111
112 fn walkdir(root: &Path) -> Vec<std::path::PathBuf> {
113 let mut out = Vec::new();
114 let mut stack = vec![root.to_path_buf()];
115 while let Some(dir) = stack.pop() {
116 let Ok(entries) = std::fs::read_dir(&dir) else {
117 continue;
118 };
119 for entry in entries.flatten() {
120 let path = entry.path();
121 if path.is_dir() {
122 stack.push(path);
123 } else {
124 out.push(path);
125 }
126 }
127 }
128 out
129 }
130
131 #[test]
132 fn collect_disables_in_push_auto_gc() {
133 let (_dir, git_dir) = bare();
134 collect(&git_dir, "now").unwrap();
135
136 for (key, expected) in [("gc.auto", "0"), ("receive.autogc", "false")] {
137 assert_eq!(
138 Git::at(&git_dir).text(&["config", "--get", key]).unwrap(),
139 expected,
140 "{key} must be turned off so gc does not run inside a push"
141 );
142 }
143 }
144
145 #[test]
146 fn an_unreachable_object_survives_the_grace_window() {
147 let (_dir, git_dir) = bare();
148 let oid = loose(&git_dir, b"unreachable but recent");
149
150 // Nothing references it, but it is newer than the grace period.
151 collect(&git_dir, "2.weeks.ago").unwrap();
152
153 let still = Command::new("git")
154 .arg("--git-dir")
155 .arg(&git_dir)
156 .args(["cat-file", "-e", &oid])
157 .status()
158 .unwrap();
159 assert!(
160 still.success(),
161 "pruning inside the grace window is how a concurrent clone gets corrupted"
162 );
163 }
164
165 #[test]
166 fn an_unreachable_object_is_pruned_once_the_window_has_passed() {
167 let (_dir, git_dir) = bare();
168 let oid = loose(&git_dir, b"unreachable and expired");
169 assert_eq!(count_loose(&git_dir), 1);
170
171 collect(&git_dir, "now").unwrap();
172
173 let still = Command::new("git")
174 .arg("--git-dir")
175 .arg(&git_dir)
176 .args(["cat-file", "-e", &oid])
177 .status()
178 .unwrap();
179 assert!(
180 !still.success(),
181 "an expired unreachable object must be reclaimed"
182 );
183 }
184
185 #[test]
186 fn a_reachable_object_is_never_pruned_however_short_the_window() {
187 let (_dir, git_dir) = bare();
188
189 let tree = loose(&git_dir, b"");
190 let tree = Command::new("git")
191 .arg("--git-dir")
192 .arg(&git_dir)
193 .args(["hash-object", "-t", "tree", "-w", "--stdin"])
194 .stdin(Stdio::null())
195 .output()
196 .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
197 .unwrap_or(tree);
198
199 let commit = Command::new("git")
200 .arg("--git-dir")
201 .arg(&git_dir)
202 .args(["commit-tree", &tree, "-m", "keep"])
203 .env("GIT_AUTHOR_NAME", "t")
204 .env("GIT_AUTHOR_EMAIL", "t@e")
205 .env("GIT_COMMITTER_NAME", "t")
206 .env("GIT_COMMITTER_EMAIL", "t@e")
207 .output()
208 .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
209 .unwrap();
210
211 Command::new("git")
212 .arg("--git-dir")
213 .arg(&git_dir)
214 .args(["update-ref", "refs/heads/main", &commit])
215 .status()
216 .unwrap();
217
218 collect(&git_dir, "now").unwrap();
219
220 assert!(Command::new("git")
221 .arg("--git-dir")
222 .arg(&git_dir)
223 .args(["cat-file", "-e", &commit])
224 .status()
225 .unwrap()
226 .success());
227 verify(&git_dir).expect("the repository must remain intact after gc");
228 }
229
230 #[test]
231 fn collect_is_idempotent() {
232 let (_dir, git_dir) = bare();
233 collect(&git_dir, "now").unwrap();
234 collect(&git_dir, "now").unwrap();
235 verify(&git_dir).unwrap();
236 }
237}