diff options
author | Kim Altintop <kim@eagain.io> | 2023-01-09 13:18:33 +0100 |
---|---|---|
committer | Kim Altintop <kim@eagain.io> | 2023-01-09 13:18:33 +0100 |
commit | d2f423521ec76406944ad83098ec33afe20c692b (patch) | |
tree | afd86bcb088eebdd61ba4e52fa666ff0f41c42a2 /src/patches |
This is it
Squashed commit of all the exploration history. Development starts here.
Signed-off-by: Kim Altintop <kim@eagain.io>
Diffstat (limited to 'src/patches')
-rw-r--r-- | src/patches/bundle.rs | 344 | ||||
-rw-r--r-- | src/patches/error.rs | 29 | ||||
-rw-r--r-- | src/patches/iter.rs | 395 | ||||
-rw-r--r-- | src/patches/notes.rs | 181 | ||||
-rw-r--r-- | src/patches/record.rs | 472 | ||||
-rw-r--r-- | src/patches/state.rs | 231 | ||||
-rw-r--r-- | src/patches/submit.rs | 574 | ||||
-rw-r--r-- | src/patches/traits.rs | 165 |
8 files changed, 2391 insertions, 0 deletions
diff --git a/src/patches/bundle.rs b/src/patches/bundle.rs new file mode 100644 index 0000000..296b24a --- /dev/null +++ b/src/patches/bundle.rs @@ -0,0 +1,344 @@ +// Copyright © 2022 Kim Altintop <kim@eagain.io> +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + fs::File, + io::{ + self, + Read, + Seek, + SeekFrom, + }, + iter, + path::{ + Path, + PathBuf, + }, +}; + +use anyhow::{ + bail, + ensure, + Context, +}; +use multipart::client::lazy::Multipart; +use sha2::{ + Digest, + Sha256, +}; +use tempfile::NamedTempFile; +use url::Url; + +use super::record::{ + self, + Encryption, +}; +use crate::{ + bundle, + io::HashWriter, + keys::Signature, + Result, +}; + +pub struct Bundle { + pub(super) header: bundle::Header, + pub(super) path: PathBuf, + pub(super) info: bundle::Info, + pub(super) encryption: Option<Encryption>, + pack_start: u64, +} + +impl Bundle { + pub fn create<P>(bundle_dir: P, repo: &git2::Repository, header: bundle::Header) -> Result<Self> + where + P: AsRef<Path>, + { + let bundle_dir = bundle_dir.as_ref(); + std::fs::create_dir_all(bundle_dir)?; + + let mut tmp = NamedTempFile::new_in(bundle_dir)?; + let info = bundle::create(&mut tmp, repo, &header)?; + let path = bundle_dir + .join(info.hash.to_string()) + .with_extension(bundle::FILE_EXTENSION); + tmp.persist(&path)?; + let mut buf = Vec::new(); + header.to_writer(&mut buf)?; + let pack_start = buf.len() as u64; + + Ok(Self { + header, + path, + info, + encryption: None, + pack_start, + }) + } + + pub fn from_fetched(bundle: bundle::Fetched) -> Result<Self> { + let (path, info) = bundle.into_inner(); + let (header, mut pack) = split(&path)?; + let pack_start = pack.offset; + let encryption = pack.encryption()?; + + Ok(Self { + header, + path, + info, + encryption, + pack_start, + }) + } + + // TODO: defer computing the checksum until needed + pub fn from_stored<P>(bundle_dir: P, expect: bundle::Expect) -> Result<Self> + where + P: AsRef<Path>, + { + let path = bundle_dir + .as_ref() + .join(expect.hash.to_string()) + .with_extension(bundle::FILE_EXTENSION); + + let (header, mut pack) = split(&path)?; + let pack_start = pack.offset; + let encryption = pack.encryption()?; + drop(pack); + let mut file = File::open(&path)?; + let mut sha2 = Sha256::new(); + + let len = io::copy(&mut file, &mut sha2)?; + let hash = header.hash(); + ensure!(expect.hash == &hash, "header hash mismatch"); + let checksum = sha2.finalize().into(); + if let Some(expect) = expect.checksum { + ensure!(expect == checksum, "claimed and actual hash differ"); + } + + let info = bundle::Info { + len, + hash, + checksum, + uris: vec![], + }; + + Ok(Self { + header, + path, + info, + encryption, + pack_start, + }) + } + + pub fn copy<R, P>(mut from: R, to: P) -> Result<Self> + where + R: Read, + P: AsRef<Path>, + { + std::fs::create_dir_all(&to)?; + let mut tmp = NamedTempFile::new_in(&to)?; + let mut out = HashWriter::new(Sha256::new(), &mut tmp); + + let len = io::copy(&mut from, &mut out)?; + let checksum = out.hash().into(); + + let (header, mut pack) = split(tmp.path())?; + let hash = header.hash(); + let pack_start = pack.offset; + let encryption = pack.encryption()?; + + let info = bundle::Info { + len, + hash, + checksum, + uris: vec![], + }; + + let path = to + .as_ref() + .join(hash.to_string()) + .with_extension(bundle::FILE_EXTENSION); + tmp.persist(&path)?; + + Ok(Self { + header, + path, + info, + encryption, + pack_start, + }) + } + + pub fn encryption(&self) -> Option<Encryption> { + self.encryption + } + + pub fn is_encrypted(&self) -> bool { + self.encryption.is_some() + } + + pub fn reader(&self) -> Result<impl io::Read> { + Ok(File::open(&self.path)?) + } + + pub fn header(&self) -> &bundle::Header { + &self.header + } + + pub fn info(&self) -> &bundle::Info { + &self.info + } + + pub fn packdata(&self) -> Result<Packdata> { + let bundle = File::open(&self.path)?; + Ok(Packdata { + offset: self.pack_start, + bundle, + }) + } + + pub fn default_location(&self) -> bundle::Location { + let uri = bundle::Uri::Relative(format!("/bundles/{}.bundle", self.info.hash)); + let id = hex::encode(Sha256::digest(uri.as_str())); + + bundle::Location { + id, + uri, + filter: None, + creation_token: None, + location: None, + } + } + + pub fn bundle_list_path(&self) -> PathBuf { + self.path.with_extension(bundle::list::FILE_EXTENSION) + } + + pub fn write_bundle_list<I>(&self, extra: I) -> Result<()> + where + I: IntoIterator<Item = bundle::Location>, + { + let mut blist = bundle::List::any(); + blist.extend( + iter::once(self.default_location()) + .chain(self.info.uris.iter().map(|url| { + let uri = bundle::Uri::Absolute(url.clone()); + let id = hex::encode(Sha256::digest(uri.as_str())); + + bundle::Location { + id, + uri, + filter: None, + creation_token: None, + location: None, + } + })) + .chain(extra), + ); + + let mut cfg = git2::Config::open(&self.bundle_list_path())?; + blist.to_config(&mut cfg)?; + + Ok(()) + } + + pub fn sign<S>(&self, signer: &mut S) -> Result<Signature> + where + S: crate::keys::Signer, + { + Ok(signer.sign(record::Heads::from(&self.header).as_slice())?) + } + + pub fn ipfs_add(&mut self, via: &Url) -> Result<Url> { + let name = format!("{}.{}", self.info.hash, bundle::FILE_EXTENSION); + let mut api = via.join("api/v0/add")?; + api.query_pairs_mut() + // FIXME: we may want this, but `rust-chunked-transfer` (used by + // `ureq`) doesn't know about trailers + // .append_pair("to-files", &name) + .append_pair("quiet", "true"); + let mpart = Multipart::new() + .add_file(name, self.path.as_path()) + .prepare()?; + + #[derive(serde::Deserialize)] + struct Response { + #[serde(rename = "Hash")] + cid: String, + } + + let Response { cid } = ureq::post(api.as_str()) + .set( + "Content-Length", + &mpart + .content_len() + .expect("zero-size bundle file?") + .to_string(), + ) + .set( + "Content-Type", + &format!("multipart/form-data; boundary={}", mpart.boundary()), + ) + .send(mpart) + .context("posting to IPFS API")? + .into_json() + .context("parsing IPFS API response")?; + + let url = Url::parse(&format!("ipfs://{cid}"))?; + self.info.uris.push(url.clone()); + + Ok(url) + } +} + +impl From<Bundle> for bundle::Info { + fn from(Bundle { info, .. }: Bundle) -> Self { + info + } +} + +fn split(bundle: &Path) -> Result<(bundle::Header, Packdata)> { + let mut bundle = File::open(bundle)?; + let header = bundle::Header::from_reader(&mut bundle)?; + let offset = bundle.stream_position()?; + let pack = Packdata { offset, bundle }; + Ok((header, pack)) +} + +pub struct Packdata { + offset: u64, + bundle: File, +} + +impl Packdata { + pub fn index(&mut self, odb: &git2::Odb) -> Result<()> { + self.bundle.seek(SeekFrom::Start(self.offset))?; + + let mut pw = odb.packwriter()?; + io::copy(&mut self.bundle, &mut pw)?; + pw.commit()?; + + Ok(()) + } + + pub fn encryption(&mut self) -> Result<Option<Encryption>> { + const PACK: &[u8] = b"PACK"; + const AGE: &[u8] = b"age-encryption.org/v1"; + const GPG: &[u8] = b"-----BEGIN PGP MESSAGE-----"; + + self.bundle.seek(SeekFrom::Start(self.offset))?; + + let mut buf = [0; 32]; + self.bundle.read_exact(&mut buf)?; + if buf.starts_with(PACK) { + Ok(None) + } else if buf.starts_with(AGE) { + Ok(Some(Encryption::Age)) + } else if buf.starts_with(GPG) { + Ok(Some(Encryption::Gpg)) + } else { + bail!("packdata does not appear to be in a known format") + } + } +} diff --git a/src/patches/error.rs b/src/patches/error.rs new file mode 100644 index 0000000..a02ed94 --- /dev/null +++ b/src/patches/error.rs @@ -0,0 +1,29 @@ +// Copyright © 2022 Kim Altintop <kim@eagain.io> +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use thiserror::Error; + +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum FromTree { + #[error("'{name}' not found in tree")] + NotFound { name: &'static str }, + + #[error("expected '{name}' to be a blob, but found {kind:?}")] + TypeMismatch { + name: &'static str, + kind: Option<git2::ObjectType>, + }, + + #[error("max blob size {max} exceeded: {found}")] + BlobSize { max: usize, found: usize }, + + #[error("type conversion from byte slice to T failed")] + TypeConversion(#[source] crate::Error), + + #[error("invalid signature")] + InvalidSignature(#[from] signature::Error), + + #[error(transparent)] + Git(#[from] git2::Error), +} diff --git a/src/patches/iter.rs b/src/patches/iter.rs new file mode 100644 index 0000000..6023247 --- /dev/null +++ b/src/patches/iter.rs @@ -0,0 +1,395 @@ +// Copyright © 2022 Kim Altintop <kim@eagain.io> +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + collections::BTreeSet, + rc::Rc, + str::FromStr, +}; + +use anyhow::anyhow; +use time::{ + OffsetDateTime, + UtcOffset, +}; + +use super::{ + notes, + record::{ + Heads, + Record, + }, + Topic, + GLOB_IT_TOPICS, + TOPIC_MERGES, +}; +use crate::{ + git::{ + self, + Refname, + EMPTY_TREE, + }, + iter, + patches::REF_IT_BUNDLES, + Result, +}; + +pub mod dropped { + use super::*; + use crate::{ + error, + patches::TOPIC_SNAPSHOTS, + }; + + pub fn topics<'a>( + repo: &'a git2::Repository, + drop_ref: &'a str, + ) -> impl Iterator<Item = Result<(Topic, git2::Oid)>> + 'a { + let topic = move |oid| -> Result<Option<(Topic, git2::Oid)>> { + let commit = repo.find_commit(oid)?; + Ok(Topic::from_commit(&commit)?.map(|topic| (topic, oid))) + }; + let init = || { + let mut walk = repo.revwalk()?; + walk.push_ref(drop_ref)?; + Ok(walk.map(|i| i.map_err(Into::into))) + }; + + iter::Iter::new(init, Some).filter_map(move |oid| oid.and_then(topic).transpose()) + } + + pub fn topic<'a>( + repo: &'a git2::Repository, + drop_ref: &'a str, + topic: &'a Topic, + ) -> impl Iterator<Item = Result<git2::Oid>> + 'a { + topics(repo, drop_ref).filter_map(move |i| { + i.map(|(top, oid)| (&top == topic).then_some(oid)) + .transpose() + }) + } + + #[allow(unused)] + pub fn merges<'a>( + repo: &'a git2::Repository, + drop_ref: &'a str, + ) -> impl Iterator<Item = Result<git2::Oid>> + 'a { + topic(repo, drop_ref, &TOPIC_MERGES) + } + + #[allow(unused)] + pub fn snapshots<'a>( + repo: &'a git2::Repository, + drop_ref: &'a str, + ) -> impl Iterator<Item = Result<git2::Oid>> + 'a { + topic(repo, drop_ref, &TOPIC_SNAPSHOTS) + } + + pub fn records<'a>( + repo: &'a git2::Repository, + drop_ref: &'a str, + ) -> impl Iterator<Item = Result<Record>> + 'a { + _records(repo, drop_ref, false) + } + + pub fn records_rev<'a>( + repo: &'a git2::Repository, + drop_ref: &'a str, + ) -> impl Iterator<Item = Result<Record>> + 'a { + _records(repo, drop_ref, true) + } + + fn _records<'a>( + repo: &'a git2::Repository, + drop_ref: &'a str, + rev: bool, + ) -> impl Iterator<Item = Result<Record>> + 'a { + let record = move |oid| -> Result<Option<Record>> { + let commit = repo.find_commit(oid)?; + match Record::from_commit(repo, &commit) { + Ok(r) => Ok(Some(r)), + Err(e) => match e.downcast_ref::<error::NotFound<&str, String>>() { + Some(error::NotFound { what: "topic", .. }) => Ok(None), + _ => Err(e), + }, + } + }; + let init = move || { + let mut walk = repo.revwalk()?; + walk.push_ref(drop_ref)?; + if rev { + walk.set_sorting(git2::Sort::REVERSE)?; + } + Ok(walk.map(|i| i.map_err(Into::into))) + }; + + iter::Iter::new(init, Some).filter_map(move |oid| oid.and_then(record).transpose()) + } +} + +pub mod unbundled { + use super::*; + + #[allow(unused)] + pub fn topics(repo: &git2::Repository) -> impl Iterator<Item = Result<Topic>> + '_ { + iter::Iter::new( + move || { + let refs = repo.references_glob(GLOB_IT_TOPICS.glob())?; + Ok(git::ReferenceNames::new(refs, Topic::from_refname)) + }, + Some, + ) + } + + pub fn topics_with_subject( + repo: &git2::Repository, + ) -> impl Iterator<Item = Result<(Topic, String)>> + '_ { + let topic_and_subject = move |refname: &str| -> Result<(Topic, String)> { + let topic = Topic::from_refname(refname)?; + let subject = find_subject(repo, refname)?; + Ok((topic, subject)) + }; + iter::Iter::new( + move || { + let refs = repo.references_glob(GLOB_IT_TOPICS.glob())?; + Ok(git::ReferenceNames::new(refs, topic_and_subject)) + }, + Some, + ) + } + + // TODO: cache this somewhere + fn find_subject(repo: &git2::Repository, topic_ref: &str) -> Result<String> { + let mut walk = repo.revwalk()?; + walk.push_ref(topic_ref)?; + walk.simplify_first_parent()?; + walk.set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE)?; + match walk.next() { + None => Ok(String::default()), + Some(oid) => { + let tree = repo.find_commit(oid?)?.tree()?; + let note = notes::Note::from_tree(repo, &tree)?; + let subj = match note { + notes::Note::Simple(n) => n + .checkpoint_kind() + .map(|k| { + match k { + notes::CheckpointKind::Merge => "Merges", + notes::CheckpointKind::Snapshot => "Snapshots", + } + .to_owned() + }) + .unwrap_or_else(|| n.subject().unwrap_or_default().to_owned()), + _ => String::default(), + }; + + Ok(subj) + }, + } + } +} + +#[derive(Eq, PartialEq, serde::Serialize)] +pub struct Subject { + pub name: String, + pub email: String, +} + +impl TryFrom<git2::Signature<'_>> for Subject { + type Error = std::str::Utf8Error; + + fn try_from(git: git2::Signature<'_>) -> std::result::Result<Self, Self::Error> { + let utf8 = |bs| std::str::from_utf8(bs).map(ToOwned::to_owned); + + let name = utf8(git.name_bytes())?; + let email = utf8(git.email_bytes())?; + + Ok(Self { name, email }) + } +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct NoteHeader { + #[serde(with = "git::serde::oid")] + pub id: git2::Oid, + pub author: Subject, + /// `Some` iff different from `author` + #[serde(skip_serializing_if = "Option::is_none")] + pub committer: Option<Subject>, + /// Committer time + #[serde(with = "time::serde::rfc3339")] + pub time: OffsetDateTime, + pub patch: Rc<PatchInfo>, + #[serde( + with = "git::serde::oid::option", + skip_serializing_if = "Option::is_none" + )] + pub in_reply_to: Option<git2::Oid>, +} + +#[derive(serde::Serialize)] +pub struct PatchInfo { + pub id: Heads, + pub tips: BTreeSet<Refname>, +} + +#[derive(serde::Serialize)] +pub struct Note { + pub header: NoteHeader, + pub message: notes::Note, +} + +pub fn topic<'a>( + repo: &'a git2::Repository, + topic: &'a Topic, +) -> impl Iterator<Item = Result<Note>> + DoubleEndedIterator + 'a { + let init = move || { + let topic_ref = topic.as_refname(); + let mut walk = repo.revwalk()?; + walk.push_ref(&topic_ref)?; + walk.set_sorting(git2::Sort::TOPOLOGICAL)?; + + fn patch_id(c: &git2::Commit) -> Result<Option<Heads>> { + let parse = || Heads::try_from(c); + let is_merge = c.tree_id() == *EMPTY_TREE; + is_merge.then(parse).transpose() + } + + fn patch_info(repo: &git2::Repository, id: Heads) -> Result<PatchInfo> { + let prefix = format!("{}/{}", REF_IT_BUNDLES, id); + let glob = format!("{prefix}/**"); + let mut iter = repo.references_glob(&glob)?; + let tips = iter + .names() + .filter_map(|i| match i { + Err(e) => Some(Err(e.into())), + Ok(name) + if name + .strip_prefix(&prefix) + .expect("glob yields prefix") + .starts_with("/it/") => + { + None + }, + Ok(name) => Refname::from_str(name) + .map_err(Into::into) + .map(Some) + .transpose(), + }) + .collect::<Result<_>>()?; + + Ok(PatchInfo { id, tips }) + } + + let mut patches: Vec<Rc<PatchInfo>> = Vec::new(); + let mut commits: Vec<(git2::Tree<'a>, NoteHeader)> = Vec::new(); + + if let Some(tip) = walk.next() { + // ensure tip is a merge + { + let tip = repo.find_commit(tip?)?; + let id = patch_id(&tip)?.ok_or_else(|| { + anyhow!("invalid topic '{topic_ref}': tip must be a merge commit") + })?; + let patch = patch_info(repo, id)?; + patches.push(Rc::new(patch)); + } + + for id in walk { + let commit = repo.find_commit(id?)?; + match patch_id(&commit)? { + Some(id) => { + let patch = patch_info(repo, id)?; + patches.push(Rc::new(patch)) + }, + None => { + let id = commit.id(); + let (author, committer) = { + let a = commit.author(); + let c = commit.committer(); + + if a.name_bytes() != c.name_bytes() + && a.email_bytes() != c.email_bytes() + { + let author = Subject::try_from(a)?; + let committer = Subject::try_from(c).map(Some)?; + + (author, committer) + } else { + (Subject::try_from(a)?, None) + } + }; + let time = { + let t = commit.time(); + let ofs = UtcOffset::from_whole_seconds(t.offset_minutes() * 60)?; + OffsetDateTime::from_unix_timestamp(t.seconds())?.replace_offset(ofs) + }; + let tree = commit.tree()?; + let patch = Rc::clone(&patches[patches.len() - 1]); + let in_reply_to = commit.parent_ids().next(); + + let header = NoteHeader { + id, + author, + committer, + time, + patch, + in_reply_to, + }; + + commits.push((tree, header)); + }, + } + } + } + + Ok(commits.into_iter().map(move |(tree, header)| { + notes::Note::from_tree(repo, &tree).map(|message| Note { header, message }) + })) + }; + + iter::Iter::new(init, Some) +} + +pub mod topic { + use crate::git::if_not_found_none; + + use super::*; + + pub(crate) fn default_reply_to( + repo: &git2::Repository, + topic: &Topic, + ) -> Result<Option<git2::Oid>> { + let topic_ref = topic.as_refname(); + if if_not_found_none(repo.refname_to_id(&topic_ref))?.is_none() { + return Ok(None); + } + + let mut walk = repo.revwalk()?; + walk.set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE)?; + walk.push_ref(&topic_ref)?; + + let first = walk + .next() + .expect("topic can't be empty, because {topic_ref} exists")?; + let mut last = first; + let mut seen = BTreeSet::<git2::Oid>::new(); + for id in walk { + let id = id?; + let commit = repo.find_commit(id)?; + if commit.tree_id() != *EMPTY_TREE { + let first_parent = commit + .parent_ids() + .next() + .expect("commit {id} must have a parent"); + if first_parent == first || !seen.contains(&first_parent) { + last = id; + } + seen.insert(id); + } + } + + Ok(Some(last)) + } +} diff --git a/src/patches/notes.rs b/src/patches/notes.rs new file mode 100644 index 0000000..b85ca64 --- /dev/null +++ b/src/patches/notes.rs @@ -0,0 +1,181 @@ +// Copyright © 2022 Kim Altintop <kim@eagain.io> +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + cmp, + collections::BTreeMap, + convert::Infallible, + io, + ops::Range, +}; + +use super::{ + error, + traits::{ + Blob, + BlobData, + TreeData, + }, +}; +use crate::{ + bundle::ObjectId, + git::Refname, +}; + +#[derive(serde::Serialize)] +#[serde(untagged)] +pub enum Note { + Simple(Simple), + Automerge(Automerge), +} + +impl Note { + pub fn from_tree<'a>(repo: &'a git2::Repository, tree: &git2::Tree<'a>) -> crate::Result<Self> { + Blob::<Simple>::from_tree(repo, tree) + .map(|Blob { content, .. }| Self::Simple(content)) + .or_else(|e| match e { + error::FromTree::NotFound { .. } => { + let Blob { content, .. } = Blob::<Automerge>::from_tree(repo, tree)?; + Ok(Self::Automerge(content)) + }, + x => Err(x.into()), + }) + } +} + +#[derive(serde::Serialize)] +pub struct Automerge(Vec<u8>); + +impl BlobData for Automerge { + type Error = Infallible; + + const MAX_BYTES: usize = 1_000_000; + + fn from_blob(data: &[u8]) -> Result<Self, Self::Error> { + Ok(Self(data.to_vec())) + } + + fn write_blob<W: io::Write>(&self, mut writer: W) -> io::Result<()> { + writer.write_all(&self.0) + } +} + +impl TreeData for Automerge { + const BLOB_NAME: &'static str = "c"; +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +pub enum Simple { + Known(Predef), + Unknown(serde_json::Map<String, serde_json::Value>), +} + +impl Simple { + pub fn new(message: String) -> Self { + Self::basic(message) + } + + pub fn basic(message: String) -> Self { + Self::Known(Predef::Basic { message }) + } + + pub fn checkpoint( + kind: CheckpointKind, + refs: BTreeMap<Refname, ObjectId>, + message: Option<String>, + ) -> Self { + Self::Known(Predef::Checkpoint { + kind, + refs, + message, + }) + } + + pub fn from_commit(repo: &git2::Repository, commit: &git2::Commit) -> crate::Result<Self> { + let tree = commit.tree()?; + let blob = Blob::from_tree(repo, &tree)?; + + Ok(blob.content) + } + + pub fn subject(&self) -> Option<&str> { + match self { + Self::Known(k) => k.subject(), + _ => None, + } + } + + pub fn is_checkpoint(&self) -> bool { + matches!(self, Self::Known(Predef::Checkpoint { .. })) + } + + pub fn checkpoint_kind(&self) -> Option<&CheckpointKind> { + match self { + Self::Known(Predef::Checkpoint { kind, .. }) => Some(kind), + _ => None, + } + } +} + +impl BlobData for Simple { + type Error = serde_json::Error; + + const MAX_BYTES: usize = 1_000_000; + + fn from_blob(data: &[u8]) -> Result<Self, Self::Error> { + serde_json::from_slice(data) + } + + fn write_blob<W: io::Write>(&self, writer: W) -> io::Result<()> { + serde_json::to_writer_pretty(writer, self).map_err(Into::into) + } +} + +impl TreeData for Simple { + const BLOB_NAME: &'static str = "m"; +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(tag = "_type")] +pub enum Predef { + #[serde(rename = "eagain.io/it/notes/basic")] + Basic { message: String }, + #[serde(rename = "eagain.io/it/notes/code-comment")] + CodeComment { loc: SourceLoc, message: String }, + #[serde(rename = "eagain.io/it/notes/checkpoint")] + Checkpoint { + kind: CheckpointKind, + refs: BTreeMap<Refname, ObjectId>, + #[serde(skip_serializing_if = "Option::is_none")] + message: Option<String>, + }, +} + +impl Predef { + pub fn subject(&self) -> Option<&str> { + let msg = match self { + Self::Basic { message } | Self::CodeComment { message, .. } => Some(message), + Self::Checkpoint { message, .. } => message.as_ref(), + }?; + let line = msg.lines().next()?; + let subj = &line[..cmp::min(72, line.len())]; + + (!subj.is_empty()).then_some(subj) + } +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct SourceLoc { + #[serde(with = "crate::git::serde::oid")] + pub file: git2::Oid, + #[serde(skip_serializing_if = "Option::is_none")] + pub line: Option<Range<usize>>, +} + +#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum CheckpointKind { + Merge, + Snapshot, +} diff --git a/src/patches/record.rs b/src/patches/record.rs new file mode 100644 index 0000000..6a95973 --- /dev/null +++ b/src/patches/record.rs @@ -0,0 +1,472 @@ +// Copyright © 2022 Kim Altintop <kim@eagain.io> +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use core::ops::Deref; +use std::{ + collections::{ + BTreeMap, + BTreeSet, + }, + fmt, + io::{ + self, + BufRead, + }, + path::{ + Path, + PathBuf, + }, + str::FromStr, +}; + +use anyhow::{ + anyhow, + bail, + ensure, + Context, +}; + +use hex::{ + FromHex, + ToHex, +}; + +use sha2::{ + Digest, + Sha256, +}; +use signature::{ + Signature as _, + Verifier, +}; + +use super::{ + traits::{ + to_tree, + BlobData, + Foldable, + TreeData, + }, + write_sharded, + Blob, + Bundle, + Topic, + BLOB_HEADS, + BLOB_META, + HTTP_HEADER_SIGNATURE, + TOPIC_MERGES, + TOPIC_SNAPSHOTS, +}; +use crate::{ + bundle, + error::NotFound, + git::{ + self, + Refname, + }, + iter::IteratorExt, + metadata::{ + self, + identity, + ContentHash, + }, +}; + +#[derive(Clone, Copy, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Heads(#[serde(with = "hex::serde")] [u8; 32]); + +impl Heads { + const TRAILER_PREFIX: &str = "Patch:"; + + pub fn from_commit(commit: &git2::Commit) -> crate::Result<Option<Self>> { + commit.message_raw_bytes().lines().try_find_map(|line| { + line? + .strip_prefix(Self::TRAILER_PREFIX) + .map(|s| Self::from_str(s.trim()).map_err(crate::Error::from)) + .transpose() + }) + } + + pub fn as_trailer(&self) -> String { + format!("{} {}", Self::TRAILER_PREFIX, self) + } +} + +impl Deref for Heads { + type Target = [u8; 32]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AsRef<[u8]> for Heads { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl From<&bundle::Header> for Heads { + fn from(h: &bundle::Header) -> Self { + let tips = h.references.values().collect::<BTreeSet<_>>(); + let mut hasher = Sha256::new(); + for sha in tips { + hasher.update(sha.as_bytes()); + } + Self(hasher.finalize().into()) + } +} + +impl TryFrom<&git2::Commit<'_>> for Heads { + type Error = crate::Error; + + fn try_from(commit: &git2::Commit) -> Result<Self, Self::Error> { + Self::from_commit(commit)?.ok_or_else(|| { + anyhow!(NotFound { + what: "patch trailer", + whence: format!("commit {}", commit.id()), + }) + }) + } +} + +impl FromStr for Heads { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + Self::from_hex(s) + } +} + +impl FromHex for Heads { + type Error = hex::FromHexError; + + fn from_hex<T: AsRef<[u8]>>(hex: T) -> Result<Self, Self::Error> { + <[u8; 32]>::from_hex(hex).map(Self) + } +} + +impl fmt::Display for Heads { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&hex::encode(self.0)) + } +} + +impl fmt::Debug for Heads { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&hex::encode(self.0)) + } +} + +impl BlobData for Heads { + type Error = <[u8; 32] as FromHex>::Error; + + const MAX_BYTES: usize = 64; + + fn from_blob(data: &[u8]) -> Result<Self, Self::Error> { + Self::from_hex(data) + } + + fn write_blob<W: io::Write>(&self, mut writer: W) -> io::Result<()> { + writer.write_all(self.encode_hex::<String>().as_bytes()) + } +} + +impl TreeData for Heads { + const BLOB_NAME: &'static str = BLOB_HEADS; +} + +impl Foldable for Heads { + fn folded_name(&self) -> String { + self.encode_hex() + } +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct Signature { + pub signer: metadata::ContentHash, + pub signature: metadata::Signature, +} + +impl From<Signature> for tiny_http::Header { + fn from(s: Signature) -> Self { + let value = format!( + "s1={}; s2={}; sd={}", + hex::encode(s.signer.sha1), + hex::encode(s.signer.sha2), + hex::encode(s.signature.as_ref()) + ); + + Self::from_bytes(HTTP_HEADER_SIGNATURE.as_bytes(), value).unwrap() + } +} + +impl TryFrom<&tiny_http::Header> for Signature { + type Error = crate::Error; + + fn try_from(hdr: &tiny_http::Header) -> Result<Self, Self::Error> { + ensure!( + hdr.field.equiv(HTTP_HEADER_SIGNATURE), + "not a {HTTP_HEADER_SIGNATURE} header" + ); + + let mut sha1: Option<[u8; 20]> = None; + let mut sha2: Option<[u8; 32]> = None; + let mut signature = None; + for part in hdr.value.as_str().split(';') { + match part.trim().split_at(2) { + ("s1", val) => { + let bytes = <[u8; 20]>::from_hex(val)?; + sha1 = Some(bytes); + }, + ("s2", val) => { + let bytes = <[u8; 32]>::from_hex(val)?; + sha2 = Some(bytes); + }, + ("sd", val) => { + let bytes = hex::decode(val)?; + signature = Some(metadata::Signature::from_bytes(&bytes)?); + }, + + _ => continue, + } + } + + let sha1 = sha1.ok_or_else(|| anyhow!("missing sha1 identity content hash"))?; + let sha2 = sha2.ok_or_else(|| anyhow!("missing sha2 identity content hash"))?; + let signature = signature.ok_or_else(|| anyhow!("missing signature bytes"))?; + + Ok(Self { + signer: metadata::ContentHash { sha1, sha2 }, + signature, + }) + } +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct Meta { + pub bundle: BundleInfo, + pub signature: Signature, +} + +impl BlobData for Meta { + type Error = serde_json::Error; + + const MAX_BYTES: usize = 100_000; + + fn from_blob(data: &[u8]) -> Result<Self, Self::Error> { + serde_json::from_slice(data) + } + + fn write_blob<W: io::Write>(&self, writer: W) -> io::Result<()> { + serde_json::to_writer_pretty(writer, self).map_err(Into::into) + } +} + +impl TreeData for Meta { + const BLOB_NAME: &'static str = BLOB_META; +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Encryption { + Age, + Gpg, +} + +impl Encryption { + pub fn as_str(&self) -> &str { + match self { + Self::Age => "age", + Self::Gpg => "gpg", + } + } +} + +impl FromStr for Encryption { + type Err = serde_json::Error; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + serde_json::from_str(s) + } +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct BundleInfo { + #[serde(flatten)] + pub info: bundle::Info, + pub prerequisites: BTreeSet<bundle::ObjectId>, + pub references: BTreeMap<Refname, bundle::ObjectId>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encryption: Option<Encryption>, +} + +impl BundleInfo { + pub fn as_expect(&self) -> bundle::Expect { + bundle::Expect::from(&self.info) + } +} + +impl From<&Bundle> for BundleInfo { + fn from(bundle: &Bundle) -> Self { + let (prerequisites, references) = { + let h = bundle.header(); + (h.prerequisites.clone(), h.references.clone()) + }; + Self { + info: bundle.info().clone(), + prerequisites, + references, + encryption: bundle.encryption(), + } + } +} + +/// Log record of a patch submission +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct Record { + pub topic: Topic, + pub heads: Heads, + pub meta: Meta, +} + +impl Record { + pub fn from_commit<'a>( + repo: &'a git2::Repository, + commit: &git2::Commit<'a>, + ) -> crate::Result<Self> { + let topic = Topic::from_commit(commit)?.ok_or_else(|| crate::error::NotFound { + what: "topic", + whence: format!("message of commit {}", commit.id()), + })?; + + let tree = commit.tree()?; + + let mut heads: Option<Heads> = None; + let mut meta: Option<Meta> = None; + + for entry in &tree { + match entry.name() { + Some(BLOB_HEADS) => { + heads = Some(Blob::<Heads>::from_entry(repo, entry)?.content); + }, + Some(BLOB_META) => { + meta = Some(Blob::<Meta>::from_entry(repo, entry)?.content); + }, + + None | Some(_) => continue, + } + } + + let whence = || format!("tree {}", tree.id()); + let heads = heads.ok_or_else(|| crate::error::NotFound { + what: BLOB_HEADS, + whence: whence(), + })?; + let meta = meta.ok_or_else(|| crate::error::NotFound { + what: BLOB_META, + whence: whence(), + })?; + + Ok(Self { topic, heads, meta }) + } + + pub fn commit<S>( + &self, + signer: &mut S, + repo: &git2::Repository, + ids: &git2::Tree, + parent: Option<&git2::Commit>, + seen: Option<&mut git2::TreeBuilder>, + ) -> crate::Result<git2::Oid> + where + S: crate::keys::Signer, + { + let tree = { + let mut tb = repo.treebuilder(parent.map(|p| p.tree()).transpose()?.as_ref())?; + tb.insert("ids", ids.id(), git2::FileMode::Tree.into())?; + to_tree(repo, &mut tb, &self.heads)?; + to_tree(repo, &mut tb, &self.meta)?; + repo.find_tree(tb.write()?)? + }; + let oid = git::commit_signed( + signer, + repo, + self.topic.as_trailer(), + &tree, + &parent.into_iter().collect::<Vec<_>>(), + )?; + + if let Some(seen) = seen { + write_sharded( + repo, + seen, + &self.heads, + tree.get_name(Heads::BLOB_NAME) + .expect("heads blob written above") + .id(), + )?; + } + + Ok(oid) + } + + pub fn signed_part(&self) -> [u8; 32] { + *self.heads + } + + pub fn verify_signature<F>(&self, mut find_id: F) -> crate::Result<()> + where + F: FnMut(&ContentHash) -> crate::Result<identity::Verified>, + { + let signed_data = self.signed_part(); + let addr = &self.meta.signature.signer; + let signature = &self.meta.signature.signature; + let id = + find_id(addr).with_context(|| format!("invalid or non-existent id at {:?}", addr))?; + for key in id.identity().keys.values() { + if key.verify(&signed_data, signature).is_ok() { + return Ok(()); + } + } + bail!("signature key not in id at {:?}", addr); + } + + pub fn bundle_info(&self) -> &BundleInfo { + &self.meta.bundle + } + + pub fn bundle_hash(&self) -> &bundle::Hash { + &self.meta.bundle.info.hash + } + + pub fn bundle_path(&self, prefix: &Path) -> PathBuf { + let mut p = prefix.join(self.bundle_hash().to_string()); + p.set_extension(bundle::FILE_EXTENSION); + p + } + + pub fn is_encrypted(&self) -> bool { + self.meta.bundle.encryption.is_some() + } + + pub fn is_snapshot(&self) -> bool { + self.topic == *TOPIC_SNAPSHOTS + } + + pub fn is_mergepoint(&self) -> bool { + self.topic == *TOPIC_MERGES + } + + /// Remove traces of a record from the given tree + pub(crate) fn remove_from(tree: &mut git2::TreeBuilder) -> crate::Result<()> { + if tree.get(Heads::BLOB_NAME)?.is_some() { + tree.remove(Heads::BLOB_NAME)?; + } + if tree.get(Meta::BLOB_NAME)?.is_some() { + tree.remove(Meta::BLOB_NAME)?; + } + + Ok(()) + } +} diff --git a/src/patches/state.rs b/src/patches/state.rs new file mode 100644 index 0000000..220971d --- /dev/null +++ b/src/patches/state.rs @@ -0,0 +1,231 @@ +// Copyright © 2022 Kim Altintop <kim@eagain.io> +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + io, + ops::Range, +}; + +use anyhow::{ + anyhow, + ensure, + Context, +}; +use log::warn; + +use super::{ + Record, + TrackingBranch, +}; +use crate::{ + git::{ + self, + if_not_found_none, + refs::{ + self, + LockedRef, + }, + Refname, + }, + keys::VerificationKey, + metadata::{ + self, + git::FromGit, + identity, + }, + Result, +}; + +/// Somewhat ad-hoc view of the tip of a drop +pub struct DropHead<'a> { + pub tip: git2::Reference<'a>, + pub ids: git2::Tree<'a>, + pub meta: metadata::drop::Verified, +} + +impl<'a> DropHead<'a> { + pub fn from_refname<S: AsRef<str>>(repo: &'a git2::Repository, name: S) -> crate::Result<Self> { + let tip = repo.find_reference(name.as_ref())?; + let root = tip.peel_to_tree()?; + let ids = root + .get_name("ids") + .ok_or_else(|| anyhow!("invalid drop: 'ids' tree not found"))? + .to_object(repo)? + .into_tree() + .map_err(|_| anyhow!("invalid drop: 'ids' tree is not a tree"))?; + let meta = metadata::Drop::from_tree(repo, &root) + .context("error loading drop metadata")? + .verified(metadata::git::find_parent(repo), |id| { + metadata::identity::find_in_tree(repo, &ids, id) + .map(|verified| verified.into_parts().1.keys) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + })?; + + Ok(Self { tip, ids, meta }) + } +} + +pub fn unbundle( + odb: &git2::Odb, + tx: &mut refs::Transaction, + ref_prefix: &str, + record: &Record, +) -> Result<Vec<(Refname, git2::Oid)>> { + let reflog = format!("it: storing head from {}", record.bundle_hash()); + + let mut updated = Vec::with_capacity(record.meta.bundle.references.len()); + for (name, oid) in &record.meta.bundle.references { + let oid = git2::Oid::try_from(oid)?; + ensure!(odb.exists(oid), "ref not actually in bundle: {oid} {name}"); + + let by_heads = unbundled_ref(ref_prefix, record, name)?; + tx.lock_ref(by_heads.clone())? + .set_target(oid, reflog.clone()); + updated.push((by_heads, oid)); + } + + Ok(updated) +} + +pub fn unbundled_ref(prefix: &str, record: &Record, name: &Refname) -> Result<Refname> { + format!( + "{}/{}/{}", + prefix.trim_matches('/'), + record.heads, + name.trim_start_matches("refs/") + ) + .try_into() + .map_err(Into::into) +} + +pub fn merge_notes( + repo: &git2::Repository, + submitter: &identity::Verified, + topics_ref: &LockedRef, + record: &Record, +) -> Result<()> { + let theirs: git2::Oid = record + .meta + .bundle + .references + .get(topics_ref.name()) + .ok_or_else(|| anyhow!("invalid record: missing '{topics_ref}'"))? + .try_into()?; + + let tree = git::empty_tree(repo)?; + let usr = repo.signature()?; + let theirs_commit = repo.find_commit(theirs)?; + match if_not_found_none(repo.find_reference(topics_ref.name()))? { + None => { + let msg = format!( + "Create topic from '{theirs}'\n\n{}", + record.heads.as_trailer() + ); + let oid = repo.commit(None, &usr, &usr, &msg, &tree, &[&theirs_commit])?; + topics_ref.set_target(oid, "it: create topic"); + }, + Some(ours_ref) => { + let ours_commit = ours_ref.peel_to_commit()?; + let ours = ours_commit.id(); + + ensure!(ours != theirs, "illegal state: theirs equals ours ({ours})"); + + let base = repo + .merge_base(ours, theirs) + .with_context(|| format!("{topics_ref}: {theirs} diverges from {ours}"))?; + let theirs_commit = repo.find_commit(theirs)?; + + verify_commit_range(repo, submitter, theirs_commit.id()..base)?; + + let msg = format!( + "Merge '{theirs}' into {}\n\n{}", + record.topic, + record.heads.as_trailer() + ); + let oid = repo.commit( + None, + &usr, + &usr, + &msg, + &tree, + &[&ours_commit, &theirs_commit], + )?; + let reflog = format!("it: auto-merge from {theirs}"); + topics_ref.set_target(oid, reflog); + }, + } + + Ok(()) +} + +pub fn update_branches( + repo: &git2::Repository, + tx: &mut refs::Transaction, + submitter: &identity::Verified, + meta: &metadata::drop::Verified, + record: &Record, +) -> Result<()> { + let branches = meta + .roles + .branches + .iter() + .filter_map(|(name, role)| role.role.ids.contains(submitter.id()).then_some(name)); + for branch in branches { + let sandboxed = match TrackingBranch::try_from(branch) { + Ok(tracking) => tracking.into_refname(), + Err(e) => { + warn!("Skipping invalid branch {branch}: {e}"); + continue; + }, + }; + + if let Some(target) = record.meta.bundle.references.get(branch) { + let target = git2::Oid::try_from(target)?; + let locked = tx.lock_ref(sandboxed.clone())?; + let reflog = format!( + "it: update tip from {} by {}", + record.bundle_hash(), + submitter.id() + ); + match if_not_found_none(repo.refname_to_id(&sandboxed))? { + Some(ours) => { + ensure!( + repo.graph_descendant_of(target, ours)?, + "checkpoint branch {branch} diverges from previously recorded tip {target}" + ); + locked.set_target(target, reflog); + }, + None => locked.set_target(target, reflog), + } + + if repo.is_bare() { + tx.lock_ref(branch.clone())? + .set_symbolic_target(sandboxed, "it: symref auto-updated branch".to_owned()); + } + } + } + + Ok(()) +} + +fn verify_commit_range( + repo: &git2::Repository, + allowed: &identity::Verified, + Range { start, end }: Range<git2::Oid>, +) -> Result<()> { + let mut walk = repo.revwalk()?; + walk.push(start)?; + walk.hide(end)?; + walk.simplify_first_parent()?; + walk.set_sorting(git2::Sort::TOPOLOGICAL)?; + for id in walk { + let pk = git::verify_commit_signature(repo, &id?)?; + let keyid = VerificationKey::from(pk).keyid(); + ensure!( + allowed.identity().keys.contains_key(&keyid), + "good signature by unknown signer" + ); + } + + Ok(()) +} diff --git a/src/patches/submit.rs b/src/patches/submit.rs new file mode 100644 index 0000000..bca428b --- /dev/null +++ b/src/patches/submit.rs @@ -0,0 +1,574 @@ +// Copyright © 2022 Kim Altintop <kim@eagain.io> +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + path::{ + Path, + PathBuf, + }, + str::FromStr, +}; + +use anyhow::{ + anyhow, + bail, + ensure, + Context, +}; +use globset::{ + Glob, + GlobBuilder, + GlobSet, + GlobSetBuilder, +}; +use log::info; +use once_cell::sync::Lazy; +use thiserror::Error; +use tiny_http::Request; +use url::Url; + +use super::{ + bundle::Bundle, + record::{ + self, + Heads, + Signature, + }, + state, + Record, + Seen, + Topic, + HTTP_HEADER_SIGNATURE, + MAX_LEN_BUNDLE, + REF_IT_BUNDLES, + REF_IT_TOPICS, + TOPIC_MERGES, +}; +use crate::{ + bundle, + git::{ + self, + if_not_found_none, + refs, + }, + metadata::{ + self, + git::{ + FromGit, + GitMeta, + META_FILE_ID, + }, + identity, + ContentHash, + Signed, + Verified, + }, + Result, +}; + +pub static GLOB_HEADS: Lazy<Glob> = Lazy::new(|| Glob::new("refs/heads/**").unwrap()); +pub static GLOB_TAGS: Lazy<Glob> = Lazy::new(|| Glob::new("refs/tags/**").unwrap()); +pub static GLOB_NOTES: Lazy<Glob> = Lazy::new(|| Glob::new("refs/notes/**").unwrap()); + +pub static GLOB_IT_TOPICS: Lazy<Glob> = Lazy::new(|| { + GlobBuilder::new(&format!("{}/*", REF_IT_TOPICS)) + .literal_separator(true) + .build() + .unwrap() +}); +pub static GLOB_IT_IDS: Lazy<Glob> = Lazy::new(|| { + GlobBuilder::new("refs/it/ids/*") + .literal_separator(true) + .build() + .unwrap() +}); +pub static GLOB_IT_BUNDLES: Lazy<Glob> = + Lazy::new(|| Glob::new(&format!("{}/**", REF_IT_BUNDLES)).unwrap()); + +pub static ALLOWED_REFS: Lazy<GlobSet> = Lazy::new(|| { + GlobSetBuilder::new() + .add(GLOB_HEADS.clone()) + .add(GLOB_TAGS.clone()) + .add(GLOB_NOTES.clone()) + .add(GLOB_IT_TOPICS.clone()) + .add(GLOB_IT_IDS.clone()) + .build() + .unwrap() +}); + +pub struct AcceptArgs<'a, S> { + /// The prefix under which to store the refs contained in the bundle + pub unbundle_prefix: &'a str, + /// The refname of the drop history + pub drop_ref: &'a str, + /// The refname anchoring the seen objects tree + pub seen_ref: &'a str, + /// The repo to operate on + pub repo: &'a git2::Repository, + /// The signer for the drop history + pub signer: &'a mut S, + /// IPFS API address + pub ipfs_api: Option<&'a Url>, + /// Options + pub options: AcceptOptions, +} + +pub struct AcceptOptions { + /// Allow bundles to convey "fat" packs, ie. packs which do not have any + /// prerequisites + /// + /// Default: false + pub allow_fat_pack: bool, + /// Allow encrypted bundles + /// + /// Default: false + pub allow_encrypted: bool, + /// Allowed ref name patterns + /// + /// Default: + /// + /// - refs/heads/** + /// - refs/tags/** + /// - refs/notes/** + /// - refs/it/topics/* + /// - refs/it/ids/* + pub allowed_refs: GlobSet, + /// Maximum number of branches the bundle is allowed to carry + /// + /// A branch is a ref which starts with `refs/heads/`. + /// + /// Default: 1 + pub max_branches: usize, + /// Maximum number of tags the bundle is allowed to carry + /// + /// A tag is a ref which starts with `refs/tags/`. + /// + /// Default: 1 + pub max_tags: usize, + /// Maximum number of git notes refs the bundle is allowed to carry + /// + /// A notes ref is a ref which starts with `refs/notes/`. + /// + /// Default: 1 + pub max_notes: usize, + /// Maximum number of refs in the bundle, considering all refs + /// + /// Default: 10, + pub max_refs: usize, + /// Maximum number of commits a bundle ref can have + /// + /// Default: 20 + pub max_commits: usize, +} + +impl Default for AcceptOptions { + fn default() -> Self { + Self { + allow_fat_pack: false, + allow_encrypted: false, + allowed_refs: ALLOWED_REFS.clone(), + max_branches: 1, + max_tags: 1, + max_notes: 1, + max_refs: 10, + max_commits: 20, + } + } +} + +pub struct Submission { + pub signature: Signature, + pub bundle: Bundle, +} + +impl Submission { + pub fn from_http<P>(bundle_dir: P, req: &mut Request) -> Result<Self> + where + P: AsRef<Path>, + { + let len = req + .body_length() + .ok_or_else(|| anyhow!("chunked body not permitted"))?; + ensure!( + len <= MAX_LEN_BUNDLE, + "submitted patch bundle exceeds {MAX_LEN_BUNDLE}", + ); + + let mut signature = None; + + for hdr in req.headers() { + if hdr.field.equiv(HTTP_HEADER_SIGNATURE) { + let sig = Signature::try_from(hdr)?; + signature = Some(sig); + break; + } + } + + #[derive(Debug, Error)] + #[error("missing header {0}")] + struct Missing(&'static str); + + let signature = signature.ok_or(Missing(HTTP_HEADER_SIGNATURE))?; + let bundle = Bundle::copy(req.as_reader(), bundle_dir)?; + + Ok(Self { signature, bundle }) + } + + pub fn submit(self, mut base_url: Url) -> Result<Record> { + base_url + .path_segments_mut() + .map_err(|()| anyhow!("invalid url"))? + .push("patches"); + let tiny_http::Header { + field: sig_hdr, + value: sig, + } = self.signature.into(); + let req = ureq::request_url("POST", &base_url) + .set("Content-Length", &self.bundle.info.len.to_string()) + .set(sig_hdr.as_str().as_str(), sig.as_str()); + let res = req.send(self.bundle.reader()?)?; + + Ok(res.into_json()?) + } + + pub fn try_accept<S>( + &mut self, + AcceptArgs { + unbundle_prefix, + drop_ref, + seen_ref, + repo, + signer, + ipfs_api, + options, + }: AcceptArgs<S>, + ) -> Result<Record> + where + S: crate::keys::Signer, + { + ensure!( + unbundle_prefix.starts_with("refs/"), + "prefix must start with 'refs/'" + ); + ensure!( + !self.bundle.is_encrypted() || options.allow_encrypted, + "encrypted bundle rejected" + ); + + let header = &self.bundle.header; + + ensure!( + matches!(header.object_format, bundle::ObjectFormat::Sha1), + "object-format {} not (yet) supported", + header.object_format + ); + ensure!( + !header.prerequisites.is_empty() || options.allow_fat_pack, + "thin pack required" + ); + ensure!( + header.references.len() <= options.max_refs, + "max number of refs exceeded" + ); + let topic = { + let mut topic: Option<Topic> = None; + + let mut heads = 0; + let mut tags = 0; + let mut notes = 0; + static GIT_IT: Lazy<GlobSet> = Lazy::new(|| { + GlobSetBuilder::new() + .add(GLOB_HEADS.clone()) + .add(GLOB_TAGS.clone()) + .add(GLOB_NOTES.clone()) + .add(GLOB_IT_TOPICS.clone()) + .build() + .unwrap() + }); + let mut matches = Vec::with_capacity(1); + for r in header.references.keys() { + let cand = globset::Candidate::new(r); + ensure!( + options.allowed_refs.is_match_candidate(&cand), + "unconventional ref rejected: {r}" + ); + GIT_IT.matches_candidate_into(&cand, &mut matches); + match &matches[..] { + [] => {}, + [0] => heads += 1, + [1] => tags += 1, + [2] => notes += 1, + [3] => { + ensure!(topic.is_none(), "more than one topic"); + match r.split('/').next_back() { + None => bail!("invalid notes '{r}': missing topic"), + Some(s) => { + let t = Topic::from_str(s).context("invalid topic")?; + topic = Some(t); + }, + } + }, + x => unreachable!("impossible match: {x:?}"), + } + } + ensure!( + heads <= options.max_branches, + "max number of git branches exceeded" + ); + ensure!(tags <= options.max_tags, "max number of git tags exceeded"); + ensure!( + notes <= options.max_notes, + "max number of git notes exceeded" + ); + + topic.ok_or_else(|| anyhow!("missing '{}'", GLOB_IT_TOPICS.glob()))? + }; + let heads = Heads::from(header); + + let mut tx = refs::Transaction::new(repo)?; + let seen_ref = tx.lock_ref(seen_ref.parse()?)?; + let seen_tree = match if_not_found_none(repo.find_reference(seen_ref.name()))? { + Some(seen) => seen.peel_to_tree()?, + None => git::empty_tree(repo)?, + }; + ensure!(!heads.in_tree(&seen_tree)?, "submission already exists"); + + // In a bare drop, indexing the pack is enough to detect missing + // prerequisites (ie. delta bases). Otherwise, or if the bundle is + // encrypted, we need to look for merge bases from the previously + // accepted patches. + if !repo.is_bare() || self.bundle.is_encrypted() { + let mut prereqs = header + .prerequisites + .iter() + .map(git2::Oid::try_from) + .collect::<std::result::Result<Vec<_>, _>>()?; + + for r in repo.references_glob(GLOB_IT_BUNDLES.glob())? { + let commit = r?.peel_to_commit()?.id(); + for (i, id) in prereqs.clone().into_iter().enumerate() { + if if_not_found_none(repo.merge_base(commit, id))?.is_some() { + prereqs.swap_remove(i); + } + } + if prereqs.is_empty() { + break; + } + } + + ensure!( + prereqs.is_empty(), + "prerequisite commits not found, try checkpointing a branch or \ + base the patch on a previous one: {}", + prereqs + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join(", ") + ); + } + + let odb = repo.odb()?; + if !self.bundle.is_encrypted() { + let mut pack = self.bundle.packdata()?; + pack.index(&odb)?; + + let prereqs = header + .prerequisites + .iter() + .map(git2::Oid::try_from) + .collect::<std::result::Result<Vec<_>, _>>()?; + let mut walk = repo.revwalk()?; + for (name, oid) in &header.references { + walk.push(oid.try_into()?)?; + for hide in &prereqs { + walk.hide(*hide)?; + } + let mut cnt = 0; + for x in &mut walk { + let _ = x?; + cnt += 1; + ensure!( + cnt <= options.max_commits, + "{name} exceeds configured max number of commits ({})", + options.max_commits + ); + } + walk.reset()?; + } + } + + if let Some(url) = ipfs_api { + let ipfs = self.bundle.ipfs_add(url)?; + info!("Published bundle to IPFS as {ipfs}"); + } + + let record = Record { + topic, + heads, + meta: record::Meta { + bundle: record::BundleInfo::from(&self.bundle), + signature: self.signature.clone(), + }, + }; + + let drop_ref = tx.lock_ref(drop_ref.parse()?)?; + let mut drop = state::DropHead::from_refname(repo, drop_ref.name())?; + ensure!( + drop.meta.roles.snapshot.threshold.get() == 1, + "threshold signatures for drop snapshots not yet supported" + ); + ensure!( + is_signer_eligible(signer, repo, &drop.ids, &drop.meta)?, + "supplied signer does not have the 'snapshot' role needed to record patches" + ); + + let submitter = { + let mut id = Identity::find(repo, &drop.ids, &self.signature.signer)?; + id.verify_signature(&record.signed_part(), &self.signature)?; + if let Some(updated) = id.update(repo, &drop.ids)? { + drop.ids = updated; + } + id.verified + }; + + let mut seen = repo.treebuilder(Some(&seen_tree))?; + let new_head = record.commit( + signer, + repo, + &drop.ids, + Some(&drop.tip.peel_to_commit()?), + Some(&mut seen), + )?; + drop_ref.set_target(new_head, format!("commit: {}", record.topic)); + seen_ref.set_target(seen.write()?, format!("it: update to record {}", new_head)); + + if !self.bundle.is_encrypted() { + state::unbundle(&odb, &mut tx, unbundle_prefix, &record)?; + let topic_ref = tx.lock_ref(record.topic.as_refname())?; + state::merge_notes(repo, &submitter, &topic_ref, &record)?; + if record.topic == *TOPIC_MERGES { + state::update_branches(repo, &mut tx, &submitter, &drop.meta, &record)?; + } + } + + tx.commit()?; + + Ok(record) + } +} + +fn is_signer_eligible<S>( + signer: &S, + repo: &git2::Repository, + ids: &git2::Tree, + meta: &Verified<metadata::Drop>, +) -> Result<bool> +where + S: crate::keys::Signer, +{ + let signer_id = metadata::KeyId::from(signer.ident()); + for id in &meta.roles.snapshot.ids { + let s = metadata::identity::find_in_tree(repo, ids, id)?; + if s.identity().keys.contains_key(&signer_id) { + return Ok(true); + } + } + + Ok(false) +} + +struct Identity { + verified: identity::Verified, + to_update: Option<Signed<metadata::Identity>>, +} + +impl Identity { + fn find(repo: &git2::Repository, ids: &git2::Tree, hash: &ContentHash) -> Result<Self> { + let find_parent = metadata::git::find_parent(repo); + + let (theirs_hash, theirs_signed, theirs) = metadata::Identity::from_content_hash( + repo, hash, + ) + .and_then(|GitMeta { hash, signed }| { + let signed_dup = signed.clone(); + let verified = signed.verified(&find_parent)?; + Ok((hash, signed_dup, verified)) + })?; + + let tree_path = PathBuf::from(theirs.id().to_string()).join(META_FILE_ID); + let newer = match if_not_found_none(ids.get_path(&tree_path))? { + None => Self { + verified: theirs, + to_update: Some(theirs_signed), + }, + Some(in_tree) if theirs_hash == in_tree.id() => Self { + verified: theirs, + to_update: None, + }, + Some(in_tree) => { + let (ours_hash, ours) = metadata::Identity::from_blob( + &repo.find_blob(in_tree.id())?, + ) + .and_then(|GitMeta { hash, signed }| { + let ours = signed.verified(&find_parent)?; + Ok((hash, ours)) + })?; + + if ours.identity().has_ancestor(&theirs_hash, &find_parent)? { + Self { + verified: ours, + to_update: None, + } + } else if theirs.identity().has_ancestor(&ours_hash, &find_parent)? { + Self { + verified: theirs, + to_update: Some(theirs_signed), + } + } else { + bail!( + "provided signer id at {} diverges from known id at {}", + theirs_hash, + ours_hash, + ); + } + }, + }; + + Ok(newer) + } + + fn verify_signature(&self, msg: &[u8], sig: &Signature) -> Result<()> { + ensure!( + self.verified.did_sign(msg, &sig.signature), + "signature not valid for current keys in id {}, provided signer at {}", + self.verified.id(), + sig.signer + ); + Ok(()) + } + + fn update<'a>( + &mut self, + repo: &'a git2::Repository, + root: &git2::Tree, + ) -> Result<Option<git2::Tree<'a>>> { + if let Some(meta) = self.to_update.take() { + let mut new_root = repo.treebuilder(Some(root))?; + let mut id_tree = repo.treebuilder(None)?; + metadata::identity::fold_to_tree(repo, &mut id_tree, meta)?; + new_root.insert( + self.verified.id().to_string().as_str(), + id_tree.write()?, + git2::FileMode::Tree.into(), + )?; + + let oid = new_root.write()?; + let tree = repo.find_tree(oid).map(Some)?; + + return Ok(tree); + } + + Ok(None) + } +} diff --git a/src/patches/traits.rs b/src/patches/traits.rs new file mode 100644 index 0000000..ef9ae61 --- /dev/null +++ b/src/patches/traits.rs @@ -0,0 +1,165 @@ +// Copyright © 2022 Kim Altintop <kim@eagain.io> +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + io, + path::{ + Path, + PathBuf, + }, +}; + +use super::error; +use crate::git::{ + self, + if_not_found_none, +}; + +pub trait BlobData: Sized { + type Error; + + const MAX_BYTES: usize; + + fn from_blob(data: &[u8]) -> Result<Self, Self::Error>; + fn write_blob<W: io::Write>(&self, writer: W) -> io::Result<()>; +} + +pub trait TreeData: BlobData { + const BLOB_NAME: &'static str; +} + +pub struct Blob<T> { + pub oid: git2::Oid, + pub content: T, +} + +impl<T> Blob<T> +where + T: TreeData, + T::Error: Into<crate::Error>, +{ + pub fn from_tree<'a>( + repo: &'a git2::Repository, + tree: &git2::Tree<'a>, + ) -> Result<Blob<T>, error::FromTree> { + use error::FromTree::NotFound; + + let entry = tree + .get_name(T::BLOB_NAME) + .ok_or(NotFound { name: T::BLOB_NAME })?; + Self::from_entry(repo, entry) + } + + pub fn from_entry<'a>( + repo: &'a git2::Repository, + entry: git2::TreeEntry<'a>, + ) -> Result<Self, error::FromTree> { + use error::FromTree::{ + BlobSize, + TypeConversion, + TypeMismatch, + }; + + let blob = entry + .to_object(repo)? + .into_blob() + .map_err(|obj| TypeMismatch { + name: T::BLOB_NAME, + kind: obj.kind(), + })?; + let sz = blob.size(); + if sz > T::MAX_BYTES { + return Err(BlobSize { + max: T::MAX_BYTES, + found: sz, + }); + } + let content = T::from_blob(blob.content()) + .map_err(Into::into) + .map_err(TypeConversion)?; + + Ok(Self { + oid: entry.id(), + content, + }) + } +} + +pub trait Foldable { + fn folded_name(&self) -> String; +} + +pub trait Seen { + fn in_odb(&self, odb: &git2::Odb) -> git::Result<bool>; + fn in_tree(&self, tree: &git2::Tree) -> git::Result<bool>; +} + +impl<T> Seen for T +where + T: BlobData + Foldable, +{ + fn in_odb(&self, odb: &git2::Odb) -> git::Result<bool> { + let hash = blob_hash(self)?; + Ok(odb.exists(hash)) + } + + fn in_tree(&self, tree: &git2::Tree) -> git::Result<bool> { + let path = shard_path(&self.folded_name()); + Ok(if_not_found_none(tree.get_path(&path))?.is_some()) + } +} + +pub fn to_tree<T: TreeData>( + repo: &git2::Repository, + tree: &mut git2::TreeBuilder, + data: &T, +) -> git::Result<()> { + tree.insert( + T::BLOB_NAME, + to_blob(repo, data)?, + git2::FileMode::Blob.into(), + )?; + Ok(()) +} + +pub fn to_blob<T: BlobData>(repo: &git2::Repository, data: &T) -> git::Result<git2::Oid> { + let mut writer = repo.blob_writer(None)?; + data.write_blob(&mut writer).map_err(|e| { + git2::Error::new( + git2::ErrorCode::GenericError, + git2::ErrorClass::Object, + e.to_string(), + ) + })?; + writer.commit() +} + +pub fn blob_hash<T: BlobData>(data: &T) -> git::Result<git2::Oid> { + let mut buf = Vec::new(); + data.write_blob(&mut buf).unwrap(); + git::blob_hash(&buf) +} + +pub fn write_sharded<F: Foldable>( + repo: &git2::Repository, + root: &mut git2::TreeBuilder, + item: &F, + blob: git2::Oid, +) -> git::Result<()> { + let name = item.folded_name(); + let (pre, suf) = name.split_at(2); + let shard = root + .get(pre)? + .map(|entry| entry.to_object(repo)) + .transpose()?; + let mut sub = repo.treebuilder(shard.as_ref().and_then(git2::Object::as_tree))?; + sub.insert(suf, blob, git2::FileMode::Blob.into())?; + root.insert(pre, sub.write()?, git2::FileMode::Tree.into())?; + + Ok(()) +} + +pub fn shard_path(name: &str) -> PathBuf { + let (pre, suf) = name.split_at(2); + Path::new(pre).join(suf) +} |