From d2f423521ec76406944ad83098ec33afe20c692b Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 9 Jan 2023 13:18:33 +0100 Subject: This is it Squashed commit of all the exploration history. Development starts here. Signed-off-by: Kim Altintop --- src/cmd/drop/bundles.rs | 32 ++++ src/cmd/drop/bundles/prune.rs | 113 +++++++++++++ src/cmd/drop/bundles/sync.rs | 276 +++++++++++++++++++++++++++++++ src/cmd/drop/edit.rs | 368 ++++++++++++++++++++++++++++++++++++++++++ src/cmd/drop/init.rs | 194 ++++++++++++++++++++++ src/cmd/drop/serve.rs | 140 ++++++++++++++++ src/cmd/drop/show.rs | 208 ++++++++++++++++++++++++ src/cmd/drop/snapshot.rs | 20 +++ src/cmd/drop/unbundle.rs | 93 +++++++++++ 9 files changed, 1444 insertions(+) create mode 100644 src/cmd/drop/bundles.rs create mode 100644 src/cmd/drop/bundles/prune.rs create mode 100644 src/cmd/drop/bundles/sync.rs create mode 100644 src/cmd/drop/edit.rs create mode 100644 src/cmd/drop/init.rs create mode 100644 src/cmd/drop/serve.rs create mode 100644 src/cmd/drop/show.rs create mode 100644 src/cmd/drop/snapshot.rs create mode 100644 src/cmd/drop/unbundle.rs (limited to 'src/cmd/drop') diff --git a/src/cmd/drop/bundles.rs b/src/cmd/drop/bundles.rs new file mode 100644 index 0000000..7c3e726 --- /dev/null +++ b/src/cmd/drop/bundles.rs @@ -0,0 +1,32 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use crate::cmd; + +mod prune; +pub use prune::{ + prune, + Prune, +}; + +mod sync; +pub use sync::{ + sync, + Sync, +}; + +#[derive(Debug, clap::Subcommand)] +#[allow(clippy::large_enum_variant)] +pub enum Bundles { + Sync(Sync), + Prune(Prune), +} + +impl Bundles { + pub fn run(self) -> cmd::Result { + match self { + Self::Sync(args) => sync(args).map(cmd::IntoOutput::into_output), + Self::Prune(args) => prune(args).map(cmd::IntoOutput::into_output), + } + } +} diff --git a/src/cmd/drop/bundles/prune.rs b/src/cmd/drop/bundles/prune.rs new file mode 100644 index 0000000..6bd984d --- /dev/null +++ b/src/cmd/drop/bundles/prune.rs @@ -0,0 +1,113 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + collections::BTreeSet, + fs, + path::PathBuf, + str::FromStr, +}; + +use clap::ValueHint; + +use crate::{ + bundle, + cfg, + cmd::{ + self, + ui::{ + info, + warn, + }, + }, + git, + patches::iter::dropped, +}; + +// TODO: +// +// - option to prune bundles made obsolete by snapshots + +#[derive(Debug, clap::Args)] +pub struct Prune { + /// Path to the drop repository + #[clap(from_global)] + git_dir: PathBuf, + /// The directory where to write the bundle to + /// + /// Unless this is an absolute path, it is treated as relative to $GIT_DIR. + #[clap( + long, + value_parser, + value_name = "DIR", + default_value_os_t = cfg::paths::bundles().to_owned(), + value_hint = ValueHint::DirPath, + )] + bundle_dir: PathBuf, + /// Name of a git ref holding the drop metadata history + /// + /// All locally tracked drops should be given, otherwise bundles might get + /// pruned which are still being referred to. + #[clap(long = "drop", value_parser, value_name = "REF")] + drop_refs: Vec, + /// Pretend to unlink, but don't + #[clap(long, value_parser)] + dry_run: bool, + /// Also remove location files (.uris) + #[clap(long, value_parser)] + remove_locations: bool, +} + +pub fn prune(args: Prune) -> cmd::Result> { + let repo = git::repo::open_bare(&args.git_dir)?; + let bundle_dir = if args.bundle_dir.is_relative() { + repo.path().join(args.bundle_dir) + } else { + args.bundle_dir + }; + + let mut seen = BTreeSet::new(); + for short in &args.drop_refs { + let drop_ref = repo.resolve_reference_from_short_name(short)?; + let ref_name = drop_ref.name().expect("drop references to be valid utf8"); + info!("Collecting bundle hashes from {ref_name} ..."); + for record in dropped::records(&repo, ref_name) { + let record = record?; + seen.insert(*record.bundle_hash()); + } + } + + info!("Traversing bundle dir {} ...", bundle_dir.display()); + let mut pruned = Vec::new(); + for entry in fs::read_dir(&bundle_dir)? { + let entry = entry?; + let path = entry.path(); + match path.extension() { + Some(ext) if ext == bundle::FILE_EXTENSION => { + let name = path.file_stem(); + match name + .and_then(|n| n.to_str()) + .and_then(|s| bundle::Hash::from_str(s).ok()) + { + Some(hash) => { + if !seen.contains(&hash) { + if !args.dry_run { + fs::remove_file(&path)?; + } + pruned.push(hash); + } + }, + None => warn!("Ignoring {}: file name not a bundle hash", path.display()), + } + }, + Some(ext) if ext == bundle::list::FILE_EXTENSION => { + if args.remove_locations { + fs::remove_file(&path)?; + } + }, + _ => warn!("Ignoring {}: missing .bundle", path.display()), + } + } + + Ok(pruned) +} diff --git a/src/cmd/drop/bundles/sync.rs b/src/cmd/drop/bundles/sync.rs new file mode 100644 index 0000000..21fd58b --- /dev/null +++ b/src/cmd/drop/bundles/sync.rs @@ -0,0 +1,276 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + borrow::Cow, + mem, + num::NonZeroUsize, + path::PathBuf, + sync::{ + Arc, + Mutex, + }, + time::{ + SystemTime, + UNIX_EPOCH, + }, +}; + +use anyhow::anyhow; +use clap::ValueHint; +use either::Either::{ + Left, + Right, +}; +use threadpool::ThreadPool; +use url::Url; + +use crate::{ + bundle, + cfg, + cmd::{ + self, + drop::Common, + ui::{ + debug, + info, + warn, + }, + }, + git::{ + self, + if_not_found_none, + }, + patches::{ + self, + iter::dropped, + record, + REF_IT_PATCHES, + }, +}; + +/// Max number of locations to store from the remote for which we don't know if +/// they'd succeed or not. +pub const MAX_UNTRIED_LOCATIONS: usize = 3; + +#[derive(Debug, clap::Args)] +pub struct Sync { + #[clap(flatten)] + common: Common, + /// The directory where to write the bundle to + /// + /// Unless this is an absolute path, it is treated as relative to $GIT_DIR. + #[clap( + long, + value_parser, + value_name = "DIR", + default_value_os_t = cfg::paths::bundles().to_owned(), + value_hint = ValueHint::DirPath, + )] + bundle_dir: PathBuf, + /// Name of the git ref holding the drop metadata history + #[clap(long = "drop", value_parser, value_name = "REF")] + drop_ref: Option, + /// Base URL to fetch from + #[clap(long, value_parser, value_name = "URL", value_hint = ValueHint::Url)] + url: Url, + /// Fetch via IPFS + #[clap( + long, + value_parser, + value_name = "URL", + value_hint = ValueHint::Url, + env = "IPFS_GATEWAY", + default_value_t = Url::parse("https://ipfs.io").unwrap(), + )] + ipfs_gateway: Url, + /// Fetch even if the bundle already exists locally + #[clap(long, value_parser)] + overwrite: bool, + /// Ignore snapshots if encountered + #[clap(long, value_parser)] + no_snapshots: bool, + /// Maximum number of concurrent downloads. Default is the number of + /// available cores. + #[clap(short, long, value_parser, default_value_t = def_jobs())] + jobs: NonZeroUsize, +} + +fn def_jobs() -> NonZeroUsize { + NonZeroUsize::new(num_cpus::get()).unwrap_or_else(|| NonZeroUsize::new(1).unwrap()) +} + +pub fn sync(args: Sync) -> cmd::Result> { + let repo = git::repo::open_bare(&args.common.git_dir)?; + let bundle_dir = if args.bundle_dir.is_relative() { + repo.path().join(args.bundle_dir) + } else { + args.bundle_dir + }; + let drop_ref = match args.drop_ref { + Some(rev) => if_not_found_none(repo.resolve_reference_from_short_name(&rev))? + .ok_or_else(|| anyhow!("no ref matching {rev} found"))? + .name() + .ok_or_else(|| anyhow!("invalid drop"))? + .to_owned(), + None => REF_IT_PATCHES.to_owned(), + }; + let base_url = args.url.join("bundles/")?; + let fetcher = Arc::new(Fetcher { + fetcher: bundle::Fetcher::default(), + bundle_dir, + base_url: base_url.clone(), + ipfs_gateway: args.ipfs_gateway, + }); + + let pool = ThreadPool::new(args.jobs.get()); + + let fetched = Arc::new(Mutex::new(Vec::new())); + let mut chasing_snaphots = false; + for record in dropped::records(&repo, &drop_ref) { + let record = record?; + let hexdig = record.bundle_hash().to_string(); + + if record.is_snapshot() { + if args.no_snapshots { + info!("Skipping snapshot bundle {hexdig}"); + continue; + } else { + chasing_snaphots = true; + } + } else if chasing_snaphots && !record.is_mergepoint() { + info!("Skipping non-snapshot bundle {hexdig}"); + continue; + } + + if !args.overwrite && record.bundle_path(&fetcher.bundle_dir).exists() { + info!("Skipping existing bundle {hexdig}"); + continue; + } + + let record::BundleInfo { + info: bundle::Info { len, hash, .. }, + prerequisites, + .. + } = record.bundle_info(); + let url = base_url.join(&hexdig)?; + + pool.execute({ + let len = *len; + let hash = *hash; + let fetched = Arc::clone(&fetched); + let fetcher = Arc::clone(&fetcher); + move || match fetcher.try_fetch(url, len, &hash) { + Ok(hash) => fetched.lock().unwrap().push(hash), + Err(e) => warn!("Download failed: {e}"), + } + }); + + if record.is_snapshot() && prerequisites.is_empty() { + info!("Full snapshot encountered, stopping here"); + break; + } + } + + pool.join(); + let fetched = { + let mut guard = fetched.lock().unwrap(); + mem::take(&mut *guard) + }; + + Ok(fetched) +} + +struct Fetcher { + fetcher: bundle::Fetcher, + bundle_dir: PathBuf, + base_url: Url, + ipfs_gateway: Url, +} + +impl Fetcher { + fn try_fetch(&self, url: Url, len: u64, hash: &bundle::Hash) -> cmd::Result { + info!("Fetching {url} ..."); + + let expect = bundle::Expect { + len, + hash, + checksum: None, + }; + let mut locations = Vec::new(); + let (fetched, origin) = self + .fetcher + .fetch(&url, &self.bundle_dir, expect) + .and_then(|resp| match resp { + Right(fetched) => Ok((fetched, url)), + Left(lst) => { + info!("{url}: response was a bundle list, trying alternate locations"); + + let mut iter = lst.bundles.into_iter(); + let mut found = None; + + for bundle::Location { uri, .. } in &mut iter { + if let Some(url) = self.url_from_uri(uri) { + if let Ok(Right(info)) = + self.fetcher.fetch(&url, &self.bundle_dir, expect) + { + found = Some((info, url)); + break; + } + } + } + + // If there are bundle uris left, remember a few + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("backwards system clock") + .as_secs(); + locations.extend( + iter + // Don't let the remote inflate the priority of + // unverified locations + .filter(|loc| loc.creation_token.map(|t| t < now).unwrap_or(true)) + // Only known protocols, relative to base url + .filter_map(|loc| { + let url = loc.uri.abs(&self.base_url).ok()?; + matches!(url.scheme(), "http" | "https" | "ipfs").then(|| { + bundle::Location { + uri: url.into_owned().into(), + ..loc + } + }) + }) + .take(MAX_UNTRIED_LOCATIONS), + ); + + found.ok_or_else(|| anyhow!("{url}: no reachable location found")) + }, + })?; + + info!("Downloaded {hash} from {origin}"); + let bundle = patches::Bundle::from_fetched(fetched)?; + bundle.write_bundle_list(locations)?; + + Ok(bundle.into()) + } + + fn url_from_uri(&self, uri: bundle::Uri) -> Option { + uri.abs(&self.base_url) + .map_err(Into::into) + .and_then(|url: Cow| -> cmd::Result { + match url.scheme() { + "http" | "https" => Ok(url.into_owned()), + "ipfs" => { + let cid = url + .host_str() + .ok_or_else(|| anyhow!("{url}: host part not an IPFS CID"))?; + let url = self.ipfs_gateway.join(&format!("/ipfs/{cid}"))?; + Ok(url) + }, + _ => Err(anyhow!("{url}: unsupported protocol")), + } + }) + .map_err(|e| debug!("discarding {}: {}", uri.as_str(), e)) + .ok() + } +} diff --git a/src/cmd/drop/edit.rs b/src/cmd/drop/edit.rs new file mode 100644 index 0000000..9103819 --- /dev/null +++ b/src/cmd/drop/edit.rs @@ -0,0 +1,368 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + iter, + path::PathBuf, +}; + +use anyhow::{ + anyhow, + ensure, +}; + +use super::{ + find_id, + Common, + Editable, +}; +use crate::{ + cfg, + cmd::{ + self, + ui::{ + self, + edit_commit_message, + edit_metadata, + info, + }, + Aborted, + }, + git::{ + self, + refs, + Refname, + }, + json, + keys::Signer, + metadata::{ + self, + git::{ + FromGit, + GitDrop, + META_FILE_ALTERNATES, + META_FILE_DROP, + META_FILE_MIRRORS, + }, + IdentityId, + Metadata, + }, + patches::{ + self, + REF_HEADS_PATCHES, + REF_IT_PATCHES, + }, +}; + +#[derive(Debug, clap::Args)] +pub struct Edit { + #[clap(flatten)] + common: Common, + /// Commit message for this edit + /// + /// Like git, $EDITOR will be invoked if not specified. + #[clap(short, long, value_parser)] + message: Option, + + #[clap(subcommand)] + cmd: Option, +} + +#[derive(Debug, clap::Subcommand)] +enum Cmd { + /// Edit the mirrors file + Mirrors, + /// Edit the alternates file + Alternates, +} + +#[derive(serde::Serialize)] +pub struct Output { + repo: PathBuf, + #[serde(rename = "ref")] + refname: Refname, + #[serde(with = "crate::git::serde::oid")] + commit: git2::Oid, +} + +pub fn edit(args: Edit) -> cmd::Result { + let Common { git_dir, id_path } = args.common; + + let repo = git::repo::open(git_dir)?; + let drop_ref = if repo.is_bare() { + REF_HEADS_PATCHES + } else { + REF_IT_PATCHES + } + .parse() + .unwrap(); + + let id_path = id_path.open_git(); + git::add_alternates(&repo, &id_path)?; + let cfg = repo.config()?.snapshot()?; + let signer = cfg::signer(&cfg, ui::askpass)?; + let signer_id = SignerIdentity::new(&signer, &repo, &cfg, &id_path)?; + let meta = metadata::Drop::from_tip(&repo, &drop_ref)?; + + let s = EditState { + repo, + id_path, + signer, + signer_id, + drop_ref, + meta, + }; + + match args.cmd { + None => s.edit_drop(args.message), + Some(Cmd::Mirrors) => s.edit_mirrors(args.message), + Some(Cmd::Alternates) => s.edit_alternates(args.message), + } +} + +struct EditState { + repo: git2::Repository, + id_path: Vec, + signer: S, + signer_id: SignerIdentity, + drop_ref: Refname, + meta: GitDrop, +} + +impl EditState { + fn edit_drop(mut self, message: Option) -> cmd::Result { + let GitDrop { + hash: parent_hash, + signed: metadata::Signed { signed: parent, .. }, + } = self.meta; + + ensure!( + self.signer_id.can_edit_drop(&parent), + "signer identity not allowed to edit the drop metadata" + ); + + let mut meta: metadata::Drop = edit_metadata(Editable::from(parent.clone()))?.try_into()?; + if meta.canonicalise()? == parent.canonicalise()? { + info!("Document unchanged"); + cmd::abort!(); + } + meta.prev = Some(parent_hash); + + let signed = Metadata::drop(&meta).sign(iter::once(&mut self.signer as &mut dyn Signer))?; + + let mut tx = refs::Transaction::new(&self.repo)?; + let drop_ref = tx.lock_ref(self.drop_ref)?; + + let parent = self + .repo + .find_reference(drop_ref.name())? + .peel_to_commit()?; + let parent_tree = parent.tree()?; + let mut root = self.repo.treebuilder(Some(&parent_tree))?; + patches::Record::remove_from(&mut root)?; + + let mut ids = self + .repo + .treebuilder(get_tree(&self.repo, &root, "ids")?.as_ref())?; + let identities = meta + .roles + .ids() + .into_iter() + .map(|id| find_id(&self.repo, &self.id_path, &id).map(|signed| (id, signed))) + .collect::, _>>()?; + for (iid, id) in identities { + let iid = iid.to_string(); + let mut tb = self + .repo + .treebuilder(get_tree(&self.repo, &ids, &iid)?.as_ref())?; + metadata::identity::fold_to_tree(&self.repo, &mut tb, id)?; + ids.insert(&iid, tb.write()?, git2::FileMode::Tree.into())?; + } + root.insert("ids", ids.write()?, git2::FileMode::Tree.into())?; + + root.insert( + META_FILE_DROP, + json::to_blob(&self.repo, &signed)?, + git2::FileMode::Blob.into(), + )?; + let tree = self.repo.find_tree(root.write()?)?; + + let msg = message.map(Ok).unwrap_or_else(|| { + edit_commit_message(&self.repo, drop_ref.name(), &parent_tree, &tree) + })?; + let commit = git::commit_signed(&mut self.signer, &self.repo, msg, &tree, &[&parent])?; + drop_ref.set_target(commit, "it: metadata edit"); + + tx.commit()?; + + Ok(Output { + repo: self.repo.path().to_owned(), + refname: drop_ref.into(), + commit, + }) + } + + pub fn edit_mirrors(mut self, message: Option) -> cmd::Result { + ensure!( + self.signer_id.can_edit_mirrors(&self.meta.signed.signed), + "signer identity not allowed to edit mirrors" + ); + + let prev = metadata::Mirrors::from_tip(&self.repo, &self.drop_ref) + .map(|m| m.signed.signed) + .or_else(|e| { + if e.is::() { + Ok(Default::default()) + } else { + Err(e) + } + })?; + let prev_canonical = prev.canonicalise()?; + let meta = edit_metadata(prev)?; + if meta.canonicalise()? == prev_canonical { + info!("Document unchanged"); + cmd::abort!(); + } + + let signed = + Metadata::mirrors(meta).sign(iter::once(&mut self.signer as &mut dyn Signer))?; + + let mut tx = refs::Transaction::new(&self.repo)?; + let drop_ref = tx.lock_ref(self.drop_ref)?; + + let parent = self + .repo + .find_reference(drop_ref.name())? + .peel_to_commit()?; + let parent_tree = parent.tree()?; + let mut root = self.repo.treebuilder(Some(&parent_tree))?; + patches::Record::remove_from(&mut root)?; + root.insert( + META_FILE_MIRRORS, + json::to_blob(&self.repo, &signed)?, + git2::FileMode::Blob.into(), + )?; + let tree = self.repo.find_tree(root.write()?)?; + + let msg = message.map(Ok).unwrap_or_else(|| { + edit_commit_message(&self.repo, drop_ref.name(), &parent_tree, &tree) + })?; + let commit = git::commit_signed(&mut self.signer, &self.repo, msg, &tree, &[&parent])?; + drop_ref.set_target(commit, "it: mirrors edit"); + + tx.commit()?; + + Ok(Output { + repo: self.repo.path().to_owned(), + refname: drop_ref.into(), + commit, + }) + } + + pub fn edit_alternates(mut self, message: Option) -> cmd::Result { + ensure!( + self.signer_id.can_edit_mirrors(&self.meta.signed.signed), + "signer identity not allowed to edit alternates" + ); + + let prev = metadata::Alternates::from_tip(&self.repo, &self.drop_ref) + .map(|m| m.signed.signed) + .or_else(|e| { + if e.is::() { + Ok(Default::default()) + } else { + Err(e) + } + })?; + let prev_canonical = prev.canonicalise()?; + let meta = edit_metadata(prev)?; + if meta.canonicalise()? == prev_canonical { + info!("Document unchanged"); + cmd::abort!(); + } + + let signed = + Metadata::alternates(meta).sign(iter::once(&mut self.signer as &mut dyn Signer))?; + + let mut tx = refs::Transaction::new(&self.repo)?; + let drop_ref = tx.lock_ref(self.drop_ref)?; + + let parent = self + .repo + .find_reference(drop_ref.name())? + .peel_to_commit()?; + let parent_tree = parent.tree()?; + let mut root = self.repo.treebuilder(Some(&parent_tree))?; + patches::Record::remove_from(&mut root)?; + root.insert( + META_FILE_ALTERNATES, + json::to_blob(&self.repo, &signed)?, + git2::FileMode::Blob.into(), + )?; + let tree = self.repo.find_tree(root.write()?)?; + + let msg = message.map(Ok).unwrap_or_else(|| { + edit_commit_message(&self.repo, drop_ref.name(), &parent_tree, &tree) + })?; + let commit = git::commit_signed(&mut self.signer, &self.repo, msg, &tree, &[&parent])?; + drop_ref.set_target(commit, "it: alternates edit"); + + tx.commit()?; + + Ok(Output { + repo: self.repo.path().to_owned(), + refname: drop_ref.into(), + commit, + }) + } +} + +fn get_tree<'a>( + repo: &'a git2::Repository, + builder: &git2::TreeBuilder, + name: &str, +) -> cmd::Result>> { + if let Some(entry) = builder.get(name)? { + return Ok(Some( + entry + .to_object(repo)? + .into_tree() + .map_err(|_| anyhow!("{name} is not a tree"))?, + )); + } + + Ok(None) +} + +struct SignerIdentity { + id: IdentityId, +} + +impl SignerIdentity { + pub fn new( + signer: &S, + repo: &git2::Repository, + cfg: &git2::Config, + id_path: &[git2::Repository], + ) -> cmd::Result { + let id = + cfg::git::identity(cfg)?.ok_or_else(|| anyhow!("signer identity not in gitconfig"))?; + let meta = find_id(repo, id_path, &id)?; + let keyid = metadata::KeyId::from(signer.ident()); + + ensure!( + meta.signed.keys.contains_key(&keyid), + "signing key {keyid} is not in identity {id}" + ); + + Ok(Self { id }) + } + + pub fn can_edit_drop(&self, parent: &metadata::Drop) -> bool { + parent.roles.root.ids.contains(&self.id) + } + + pub fn can_edit_mirrors(&self, parent: &metadata::Drop) -> bool { + parent.roles.mirrors.ids.contains(&self.id) + } +} diff --git a/src/cmd/drop/init.rs b/src/cmd/drop/init.rs new file mode 100644 index 0000000..b843255 --- /dev/null +++ b/src/cmd/drop/init.rs @@ -0,0 +1,194 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + iter, + num::NonZeroUsize, + path::PathBuf, +}; + +use anyhow::{ + anyhow, + ensure, +}; + +use super::{ + find_id, + Common, + Editable, +}; +use crate::{ + cfg, + cmd::{ + self, + args::Refname, + ui::{ + self, + edit_metadata, + }, + }, + git::{ + self, + if_not_found_none, + refs, + }, + json, + metadata::{ + self, + git::META_FILE_DROP, + Metadata, + }, + patches::{ + REF_HEADS_PATCHES, + REF_IT_PATCHES, + }, +}; + +#[derive(Debug, clap::Args)] +pub struct Init { + #[clap(flatten)] + common: Common, + /// A description for this drop instance, max. 128 characters + #[clap(long, value_parser, value_name = "STRING")] + description: metadata::drop::Description, + /// If the repository does not already exist, initialise it as non-bare + /// + /// A drop is usually initialised inside an already existing git repository, + /// or as a standalone drop repository. The latter is advisable for serving + /// over the network. + /// + /// When init is given a directory which does not already exist, it is + /// assumed that a standalone drop should be created, and thus the + /// repository is initialised as bare. This behaviour can be overridden + /// by --no-bare. + #[clap(long, value_parser)] + no_bare: bool, +} + +#[derive(serde::Serialize)] +pub struct Output { + repo: PathBuf, + #[serde(rename = "ref")] + refname: Refname, + #[serde(with = "crate::git::serde::oid")] + commit: git2::Oid, +} + +pub fn init(args: Init) -> cmd::Result { + let Common { git_dir, id_path } = args.common; + let drop_ref: Refname = REF_IT_PATCHES.parse().unwrap(); + + let repo = git::repo::open_or_init( + git_dir, + git::repo::InitOpts { + bare: !args.no_bare, + description: "`it` drop", + initial_head: &drop_ref, + }, + )?; + + let mut tx = refs::Transaction::new(&repo)?; + let drop_ref = tx.lock_ref(drop_ref)?; + ensure!( + if_not_found_none(repo.refname_to_id(drop_ref.name()))?.is_none(), + "{} already exists", + drop_ref + ); + + let id_path = id_path.open_git(); + git::add_alternates(&repo, &id_path)?; + + let cfg = repo.config()?.snapshot()?; + let mut signer = cfg::signer(&cfg, ui::askpass)?; + let signer_id = { + let iid = + cfg::git::identity(&cfg)?.ok_or_else(|| anyhow!("signer identity not in gitconfig"))?; + let id = find_id(&repo, &id_path, &iid)?; + let keyid = metadata::KeyId::from(signer.ident()); + ensure!( + id.signed.keys.contains_key(&keyid), + "signing key {keyid} is not in identity {iid}" + ); + + iid + }; + + let default = { + let default_role = metadata::drop::Role { + ids: [signer_id].into(), + threshold: NonZeroUsize::new(1).unwrap(), + }; + let default_branch = cfg::git::default_branch(&cfg)?; + + metadata::Drop { + spec_version: crate::SPEC_VERSION, + description: args.description, + prev: None, + custom: Default::default(), + roles: metadata::drop::Roles { + root: default_role.clone(), + snapshot: default_role.clone(), + mirrors: default_role.clone(), + branches: [( + default_branch, + metadata::drop::Annotated { + role: default_role, + description: metadata::drop::Description::try_from( + "the default branch".to_owned(), + ) + .unwrap(), + }, + )] + .into(), + }, + } + }; + let meta: metadata::Drop = edit_metadata(Editable::from(default))?.try_into()?; + ensure!( + meta.roles.root.ids.contains(&signer_id), + "signing identity {signer_id} is lacking the drop role required to sign the metadata" + ); + let signed = Metadata::drop(&meta).sign(iter::once(&mut signer))?; + + let mut root = repo.treebuilder(None)?; + let mut ids = repo.treebuilder(None)?; + let identities = meta + .roles + .ids() + .into_iter() + .map(|id| find_id(&repo, &id_path, &id).map(|signed| (id, signed))) + .collect::, _>>()?; + for (iid, id) in identities { + let iid = iid.to_string(); + let mut tb = repo.treebuilder(None)?; + metadata::identity::fold_to_tree(&repo, &mut tb, id)?; + ids.insert(&iid, tb.write()?, git2::FileMode::Tree.into())?; + } + root.insert("ids", ids.write()?, git2::FileMode::Tree.into())?; + root.insert( + META_FILE_DROP, + json::to_blob(&repo, &signed)?, + git2::FileMode::Blob.into(), + )?; + let tree = repo.find_tree(root.write()?)?; + let msg = format!("Create drop '{}'", meta.description); + let commit = git::commit_signed(&mut signer, &repo, msg, &tree, &[])?; + + if repo.is_bare() { + // Arrange refs to be `git-clone`-friendly + let heads_patches = tx.lock_ref(REF_HEADS_PATCHES.parse()?)?; + heads_patches.set_target(commit, "it: create"); + drop_ref.set_symbolic_target(heads_patches.name().clone(), String::new()); + repo.set_head(heads_patches.name())?; + } else { + drop_ref.set_target(commit, "it: create"); + } + + tx.commit()?; + + Ok(Output { + repo: repo.path().to_owned(), + refname: drop_ref.into(), + commit, + }) +} diff --git a/src/cmd/drop/serve.rs b/src/cmd/drop/serve.rs new file mode 100644 index 0000000..7540d58 --- /dev/null +++ b/src/cmd/drop/serve.rs @@ -0,0 +1,140 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + fs::File, + io::Read, + path::PathBuf, + str::FromStr, +}; + +use clap::ValueHint; +use url::Url; + +use super::Common; +use crate::{ + cfg, + cmd::{ + self, + args::Refname, + }, + http, + patches::{ + REF_IT_BUNDLES, + REF_IT_PATCHES, + REF_IT_SEEN, + }, +}; + +#[derive(Debug, clap::Args)] +pub struct Serve { + #[clap(flatten)] + common: Common, + /// The directory where to write the bundle to + /// + /// Unless this is an absolute path, it is treated as relative to $GIT_DIR. + #[clap( + long, + value_parser, + value_name = "DIR", + default_value_os_t = cfg::paths::bundles().to_owned(), + value_hint = ValueHint::DirPath, + )] + bundle_dir: PathBuf, + /// Ref prefix under which to store the refs contained in patch bundles + #[clap( + long, + value_parser, + value_name = "REF", + default_value_t = Refname::from_str(REF_IT_BUNDLES).unwrap() + )] + unbundle_prefix: Refname, + /// The refname anchoring the seen objects tree + #[clap( + long, + value_parser, + value_name = "REF", + default_value_t = Refname::from_str(REF_IT_SEEN).unwrap() + )] + seen_ref: Refname, + /// 'host:port' to listen on + #[clap( + long, + value_parser, + value_name = "HOST:PORT", + default_value = "127.0.0.1:8084" + )] + listen: String, + /// Number of threads to use for the server + /// + /// If not set, the number of available cores is used. + #[clap(long, value_parser, value_name = "INT")] + threads: Option, + /// PEM-encoded TLS certificate + /// + /// Requires 'tls-key'. If not set (the default), the server will not use + /// TLS. + #[clap( + long, + value_parser, + value_name = "FILE", + requires = "tls_key", + value_hint = ValueHint::FilePath + )] + tls_cert: Option, + /// PEM-encoded TLS private key + /// + /// Requires 'tls-cert'. If not set (the default), the server will not use + /// TLS. + #[clap( + long, + value_parser, + value_name = "FILE", + requires = "tls_cert", + value_hint = ValueHint::FilePath + )] + tls_key: Option, + /// IPFS API to publish received patch bundle to + #[clap( + long, + value_parser, + value_name = "URL", + value_hint = ValueHint::Url, + )] + ipfs_api: Option, +} + +#[derive(serde::Serialize)] +pub struct Output; + +pub fn serve(args: Serve) -> cmd::Result { + let tls = args + .tls_cert + .map(|cert_path| -> cmd::Result { + let mut certificate = Vec::new(); + let mut private_key = Vec::new(); + File::open(cert_path)?.read_to_end(&mut certificate)?; + File::open(args.tls_key.expect("presence of 'tls-key' ensured by clap"))? + .read_to_end(&mut private_key)?; + + Ok(http::SslConfig { + certificate, + private_key, + }) + }) + .transpose()?; + + http::serve( + args.listen, + http::Options { + git_dir: args.common.git_dir, + bundle_dir: args.bundle_dir, + unbundle_prefix: args.unbundle_prefix.into(), + drop_ref: REF_IT_PATCHES.into(), + seen_ref: args.seen_ref.into(), + threads: args.threads, + tls, + ipfs_api: args.ipfs_api, + }, + ) +} diff --git a/src/cmd/drop/show.rs b/src/cmd/drop/show.rs new file mode 100644 index 0000000..e3fdcfc --- /dev/null +++ b/src/cmd/drop/show.rs @@ -0,0 +1,208 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + collections::BTreeMap, + io, + path::PathBuf, +}; + +use anyhow::Context; + +use super::{ + Common, + META_FILE_ALTERNATES, + META_FILE_MIRRORS, +}; +use crate::{ + cmd::{ + self, + util::args::Refname, + FromGit as _, + GitAlternates, + GitDrop, + GitMirrors, + }, + git, + metadata::{ + self, + ContentHash, + IdentityId, + KeySet, + }, + patches::REF_IT_PATCHES, +}; + +#[derive(Debug, clap::Args)] +pub struct Show { + #[clap(flatten)] + common: Common, + /// Name of the git ref holding the drop metadata history + #[clap( + long = "drop", + value_parser, + value_name = "REF", + default_value_t = REF_IT_PATCHES.parse().unwrap(), + )] + drop_ref: Refname, +} + +#[derive(serde::Serialize)] +pub struct Output { + repo: PathBuf, + refname: Refname, + drop: Data, + #[serde(skip_serializing_if = "Option::is_none")] + mirrors: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + alternates: Option>, +} + +#[derive(serde::Serialize)] +pub struct Data { + hash: ContentHash, + status: Status, + json: T, +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum Status { + Verified, + #[serde(with = "crate::serde::display")] + Invalid(metadata::error::Verification), +} + +impl From> for Status { + fn from(r: Result<(), metadata::error::Verification>) -> Self { + r.map(|()| Self::Verified).unwrap_or_else(Self::Invalid) + } +} + +pub fn show(args: Show) -> cmd::Result { + let Common { git_dir, .. } = args.common; + let drop_ref = args.drop_ref; + + let repo = git::repo::open(git_dir)?; + + let GitDrop { + hash, + signed: metadata::Signed { + signed: drop, + signatures, + }, + } = metadata::Drop::from_tip(&repo, &drop_ref)?; + + let mut signer_cache = SignerCache::new(&repo, &drop_ref)?; + let status = drop + .verify( + &signatures, + cmd::find_parent(&repo), + find_signer(&mut signer_cache), + ) + .into(); + + let mut mirrors = None; + let mut alternates = None; + + let tree = repo.find_reference(&drop_ref)?.peel_to_commit()?.tree()?; + if let Some(entry) = tree.get_name(META_FILE_MIRRORS) { + let blob = entry.to_object(&repo)?.peel_to_blob()?; + let GitMirrors { hash, signed } = metadata::Mirrors::from_blob(&blob)?; + let status = drop + .verify_mirrors(&signed, find_signer(&mut signer_cache)) + .into(); + + mirrors = Some(Data { + hash, + status, + json: signed.signed, + }); + } + + if let Some(entry) = tree.get_name(META_FILE_ALTERNATES) { + let blob = entry.to_object(&repo)?.peel_to_blob()?; + let GitAlternates { hash, signed } = metadata::Alternates::from_blob(&blob)?; + let status = drop + .verify_alternates(&signed, find_signer(&mut signer_cache)) + .into(); + + alternates = Some(Data { + hash, + status, + json: signed.signed, + }); + } + + Ok(Output { + repo: repo.path().to_owned(), + refname: drop_ref, + drop: Data { + hash, + status, + json: drop, + }, + mirrors, + alternates, + }) +} + +struct SignerCache<'a> { + repo: &'a git2::Repository, + root: git2::Tree<'a>, + keys: BTreeMap>, +} + +impl<'a> SignerCache<'a> { + pub(self) fn new(repo: &'a git2::Repository, refname: &Refname) -> git::Result { + let root = { + let id = repo + .find_reference(refname)? + .peel_to_tree()? + .get_name("ids") + .ok_or_else(|| { + git2::Error::new( + git2::ErrorCode::NotFound, + git2::ErrorClass::Tree, + "'ids' tree not found", + ) + })? + .id(); + repo.find_tree(id)? + }; + let keys = BTreeMap::new(); + + Ok(Self { repo, root, keys }) + } +} + +fn find_signer<'a>( + cache: &'a mut SignerCache, +) -> impl FnMut(&IdentityId) -> io::Result> + 'a { + fn go( + repo: &git2::Repository, + root: &git2::Tree, + keys: &mut BTreeMap>, + id: &IdentityId, + ) -> cmd::Result> { + match keys.get(id) { + Some(keys) => Ok(keys.clone()), + None => { + let (id, verified) = metadata::identity::find_in_tree(repo, root, id) + .with_context(|| format!("identity {id} failed to verify"))? + .into_parts(); + keys.insert(id, verified.keys.clone()); + Ok(verified.keys) + }, + } + } + + |id| go(cache.repo, &cache.root, &mut cache.keys, id).map_err(as_io) +} + +fn as_io(e: E) -> io::Error +where + E: Into>, +{ + io::Error::new(io::ErrorKind::Other, e) +} diff --git a/src/cmd/drop/snapshot.rs b/src/cmd/drop/snapshot.rs new file mode 100644 index 0000000..b1348d3 --- /dev/null +++ b/src/cmd/drop/snapshot.rs @@ -0,0 +1,20 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use crate::{ + cmd::{ + self, + patch, + }, + patches, +}; + +#[derive(Debug, clap::Args)] +pub struct Snapshot { + #[clap(flatten)] + common: patch::Common, +} + +pub fn snapshot(Snapshot { common }: Snapshot) -> cmd::Result { + patch::create(patch::Kind::Snapshot { common }) +} diff --git a/src/cmd/drop/unbundle.rs b/src/cmd/drop/unbundle.rs new file mode 100644 index 0000000..a9c9f77 --- /dev/null +++ b/src/cmd/drop/unbundle.rs @@ -0,0 +1,93 @@ +// Copyright © 2022 Kim Altintop +// SPDX-License-Identifier: GPL-2.0-only WITH openvpn-openssl-exception + +use std::{ + collections::BTreeMap, + path::PathBuf, +}; + +use anyhow::anyhow; +use clap::ValueHint; + +use crate::{ + cmd, + git::{ + self, + if_not_found_none, + refs, + Refname, + }, + patches::{ + self, + iter::dropped, + Bundle, + REF_IT_BUNDLES, + REF_IT_PATCHES, + }, + paths, +}; + +// TODO: +// +// - require drop metadata verification +// - abort if existing ref would be set to a different target (or --force) +// - honour snapshots +// + +#[derive(Debug, clap::Args)] +pub struct Unbundle { + #[clap(from_global)] + git_dir: PathBuf, + /// The directory where to write the bundle to + /// + /// Unless this is an absolute path, it is treated as relative to $GIT_DIR. + #[clap( + long, + value_parser, + value_name = "DIR", + default_value_os_t = paths::bundles().to_owned(), + value_hint = ValueHint::DirPath, + )] + bundle_dir: PathBuf, + /// The drop history to find the topic in + #[clap(value_parser)] + drop: Option, +} + +#[derive(serde::Serialize)] +pub struct Output { + updated: BTreeMap, +} + +pub fn unbundle(args: Unbundle) -> cmd::Result { + let repo = git::repo::open(&args.git_dir)?; + let bundle_dir = if args.bundle_dir.is_relative() { + repo.path().join(args.bundle_dir) + } else { + args.bundle_dir + }; + let drop = match args.drop { + Some(rev) => if_not_found_none(repo.resolve_reference_from_short_name(&rev))? + .ok_or_else(|| anyhow!("no ref matching {rev} found"))? + .name() + .ok_or_else(|| anyhow!("invalid drop"))? + .to_owned(), + None => REF_IT_PATCHES.to_owned(), + }; + + let odb = repo.odb()?; + let mut tx = refs::Transaction::new(&repo)?; + let mut up = BTreeMap::new(); + for rec in dropped::records_rev(&repo, &drop) { + let rec = rec?; + let bundle = Bundle::from_stored(&bundle_dir, rec.bundle_info().as_expect())?; + bundle.packdata()?.index(&odb)?; + let updated = patches::unbundle(&odb, &mut tx, REF_IT_BUNDLES, &rec)?; + for (name, oid) in updated { + up.insert(name, oid.into()); + } + } + tx.commit()?; + + Ok(Output { updated: up }) +} -- cgit v1.2.3