cybercyst/jj
1
0
Fork 0
mirror of https://github.com/martinvonz/jj.git synced 2025-05-05 15:32:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

tree_builder: write independent trees concurrently

Browse Source 
This should help high-latency backends. I don't think we'll notice it
at Google because we hide the latency by assuming that the write will
succeed and then we flush it in the background.
This commit is contained in:
Martin von Zweigbergk 2025-04-20 00:13:02 -07:00
parent 291dd567ef
commit 6ba587c5cb
No known key found for this signature in database
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/src/tree_builder.rs
View File

@ -19,6 +19,7 @@ use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
use std::sync::Arc; use std::sync::Arc;
use futures::future::try_join_all;
use pollster::FutureExt as _; use pollster::FutureExt as _;
use crate::backend; use crate::backend;
@ -125,12 +126,13 @@ impl TreeBuilder {
} }
let mut new_unblocked = HashSet::new(); let mut new_unblocked = HashSet::new();
// TODO: Writing trees concurrently should help on high-latency backends let mut writes = vec![];
let mut dir_writes = vec![];
for dir in unblocked { for dir in unblocked {
let tree = trees_to_write.remove(dir).unwrap(); let tree = trees_to_write.remove(dir).unwrap();
let (parent, basename) = dir.split().unwrap(); let (parent, basename) = dir.split().unwrap();
let parent_tree = trees_to_write.get_mut(parent).unwrap();
if tree.is_empty() { if tree.is_empty() {
let parent_tree = trees_to_write.get_mut(parent).unwrap();
if let Some(TreeValue::Tree(_)) = parent_tree.value(basename) { if let Some(TreeValue::Tree(_)) = parent_tree.value(basename) {
parent_tree.remove(basename); parent_tree.remove(basename);
} else { } else {
@ -138,8 +140,8 @@ impl TreeBuilder {
// above) // above)
} }
} else { } else {
let tree = store.write_tree(&dir, tree).block_on()?; writes.push(store.write_tree(&dir, tree));
parent_tree.set(basename.to_owned(), TreeValue::Tree(tree.id().clone())); dir_writes.push(dir);
} }
let parent_blocked_by = blocked_by.get_mut(parent).unwrap(); let parent_blocked_by = blocked_by.get_mut(parent).unwrap();
@ -149,6 +151,14 @@ impl TreeBuilder {
new_unblocked.insert(parent); new_unblocked.insert(parent);
} }
} }
let written = try_join_all(writes).block_on()?;
for (dir, tree) in dir_writes.iter().zip(written) {
let (parent, basename) = dir.split().unwrap();
let parent_tree = trees_to_write.get_mut(parent).unwrap();
parent_tree.set(basename.to_owned(), TreeValue::Tree(tree.id().clone()));
}
unblocked = new_unblocked; unblocked = new_unblocked;
} }