From ec3bc7dd7edf645bd262073448cb30511f7b205a Mon Sep 17 00:00:00 2001 From: Stefano Fontana Date: Sat, 14 Dec 2024 16:27:14 +0100 Subject: [PATCH 1/8] refactoring downloading system to accomodate multiple downloader types. Moved http downloads into respective implementation --- crates/typst-cli/src/download.rs | 8 +- crates/typst-kit/Cargo.toml | 4 +- crates/typst-kit/src/lib.rs | 2 +- crates/typst-kit/src/package.rs | 116 ++++++----------- crates/typst-kit/src/package_downloads/git.rs | 0 .../http.rs} | 100 ++++++++++----- crates/typst-kit/src/package_downloads/mod.rs | 121 ++++++++++++++++++ crates/typst-syntax/src/package.rs | 23 +++- 8 files changed, 261 insertions(+), 113 deletions(-) create mode 100644 crates/typst-kit/src/package_downloads/git.rs rename crates/typst-kit/src/{download.rs => package_downloads/http.rs} (73%) create mode 100644 crates/typst-kit/src/package_downloads/mod.rs diff --git a/crates/typst-cli/src/download.rs b/crates/typst-cli/src/download.rs index ca1e539d..b715bdd7 100644 --- a/crates/typst-cli/src/download.rs +++ b/crates/typst-cli/src/download.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; use codespan_reporting::term; use codespan_reporting::term::termcolor::WriteColor; use typst::utils::format_duration; -use typst_kit::download::{DownloadState, Downloader, Progress}; +use typst_kit::package_downloads::{DownloadState, Downloader, Progress}; use crate::terminal::{self, TermOut}; use crate::ARGS; @@ -43,11 +43,7 @@ impl Progress for PrintDownload { /// Returns a new downloader. pub fn downloader() -> Downloader { - let user_agent = concat!("typst/", env!("CARGO_PKG_VERSION")); - match ARGS.cert.clone() { - Some(cert) => Downloader::with_path(user_agent, cert), - None => Downloader::new(user_agent), - } + Downloader::new(ARGS.cert.clone()) } /// Compile and format several download statistics and make and attempt at diff --git a/crates/typst-kit/Cargo.toml b/crates/typst-kit/Cargo.toml index 266eba0b..666b0dc1 100644 --- a/crates/typst-kit/Cargo.toml +++ b/crates/typst-kit/Cargo.toml @@ -38,7 +38,9 @@ default = ["fonts", "packages"] fonts = ["dep:fontdb", "fontdb/memmap", "fontdb/fontconfig"] # Add generic downloading utilities -downloads = ["dep:env_proxy", "dep:native-tls", "dep:ureq", "dep:openssl"] +downloads = ["downloads_http", "downloads_git"] +downloads_http = ["dep:env_proxy", "dep:native-tls", "dep:ureq", "dep:openssl"] +downloads_git = [] # Add package downloading utilities, implies `downloads` packages = ["downloads", "dep:dirs", "dep:flate2", "dep:tar"] diff --git a/crates/typst-kit/src/lib.rs b/crates/typst-kit/src/lib.rs index 956339b8..6c2c3e5b 100644 --- a/crates/typst-kit/src/lib.rs +++ b/crates/typst-kit/src/lib.rs @@ -20,7 +20,7 @@ //! `downloads` feature flag. #[cfg(feature = "downloads")] -pub mod download; +pub mod package_downloads; #[cfg(feature = "fonts")] pub mod fonts; #[cfg(feature = "packages")] diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs index e7eb71ee..440dbef9 100644 --- a/crates/typst-kit/src/package.rs +++ b/crates/typst-kit/src/package.rs @@ -1,22 +1,13 @@ //! Download and unpack packages and package indices. - -use std::fs; use std::path::{Path, PathBuf}; use ecow::eco_format; use once_cell::sync::OnceCell; -use typst_library::diag::{bail, PackageError, PackageResult, StrResult}; +use typst_library::diag::{PackageError, PackageResult, StrResult}; use typst_syntax::package::{ PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec, }; - -use crate::download::{Downloader, Progress}; - -/// The default Typst registry. -pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org"; - -/// The public namespace in the default Typst registry. -pub const DEFAULT_NAMESPACE: &str = "preview"; +use crate::package_downloads::{Downloader, PackageDownloader, Progress}; /// The default packages sub directory within the package and package cache paths. pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages"; @@ -74,25 +65,27 @@ impl PackageStorage { ) -> PackageResult { let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version); + // check the package_path for the package directory. if let Some(packages_dir) = &self.package_path { let dir = packages_dir.join(&subdir); if dir.exists() { + // no need to download, already in the path. return Ok(dir); } } + // package was not in the package_path. check if it has been cached if let Some(cache_dir) = &self.package_cache_path { let dir = cache_dir.join(&subdir); if dir.exists() { + //package was cached, so return the cached directory return Ok(dir); } // Download from network if it doesn't exist yet. - if spec.namespace == DEFAULT_NAMESPACE { - self.download_package(spec, &dir, progress)?; - if dir.exists() { - return Ok(dir); - } + self.download_package(spec, &dir, progress)?; + if dir.exists() { + return Ok(dir); } } @@ -104,47 +97,36 @@ impl PackageStorage { &self, spec: &VersionlessPackageSpec, ) -> StrResult { - if spec.namespace == DEFAULT_NAMESPACE { - // For `DEFAULT_NAMESPACE`, download the package index and find the latest - // version. - self.download_index()? - .iter() - .filter(|package| package.name == spec.name) - .map(|package| package.version) - .max() - .ok_or_else(|| eco_format!("failed to find package {spec}")) - } else { - // For other namespaces, search locally. We only search in the data - // directory and not the cache directory, because the latter is not - // intended for storage of local packages. - let subdir = format!("{}/{}", spec.namespace, spec.name); - self.package_path - .iter() - .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok()) - .flatten() - .filter_map(|entry| entry.ok()) - .map(|entry| entry.path()) - .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok()) - .max() - .ok_or_else(|| eco_format!("please specify the desired version")) + + // Same logical flow as per package download. Check package path, then check online. + // Do not check in the data directory because the latter is not intended for storage + // of local packages. + let subdir = format!("{}/{}", spec.namespace, spec.name); + let res = self.package_path + .iter() + .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok()) + .flatten() + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok()) + .max(); + + if let Some(version) = res { + return Ok(version); } + + self.download_index(spec)? + .iter() + .filter(|package| package.name == spec.name) + .map(|package| package.version) + .max() + .ok_or_else(|| eco_format!("failed to find package {spec}")) } /// Download the package index. The result of this is cached for efficiency. - pub fn download_index(&self) -> StrResult<&[PackageInfo]> { + pub fn download_index(&self, spec: &VersionlessPackageSpec) -> StrResult<&[PackageInfo]> { self.index - .get_or_try_init(|| { - let url = format!("{DEFAULT_REGISTRY}/{DEFAULT_NAMESPACE}/index.json"); - match self.downloader.download(&url) { - Ok(response) => response.into_json().map_err(|err| { - eco_format!("failed to parse package index: {err}") - }), - Err(ureq::Error::Status(404, _)) => { - bail!("failed to fetch package index (not found)") - } - Err(err) => bail!("failed to fetch package index ({err})"), - } - }) + .get_or_try_init(|| self.downloader.download_index(spec)) .map(AsRef::as_ref) } @@ -158,31 +140,15 @@ impl PackageStorage { package_dir: &Path, progress: &mut dyn Progress, ) -> PackageResult<()> { - assert_eq!(spec.namespace, DEFAULT_NAMESPACE); - - let url = format!( - "{DEFAULT_REGISTRY}/{DEFAULT_NAMESPACE}/{}-{}.tar.gz", - spec.name, spec.version - ); - - let data = match self.downloader.download_with_progress(&url, progress) { - Ok(data) => data, - Err(ureq::Error::Status(404, _)) => { + match self.downloader.download(spec, package_dir, progress) { + Err(PackageError::NotFound(spec)) => { if let Ok(version) = self.determine_latest_version(&spec.versionless()) { - return Err(PackageError::VersionNotFound(spec.clone(), version)); + Err(PackageError::VersionNotFound(spec.clone(), version)) } else { - return Err(PackageError::NotFound(spec.clone())); + Err(PackageError::NotFound(spec.clone())) } - } - Err(err) => { - return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))) - } - }; - - let decompressed = flate2::read::GzDecoder::new(data.as_slice()); - tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| { - fs::remove_dir_all(package_dir).ok(); - PackageError::MalformedArchive(Some(eco_format!("{err}"))) - }) + }, + val => val + } } } diff --git a/crates/typst-kit/src/package_downloads/git.rs b/crates/typst-kit/src/package_downloads/git.rs new file mode 100644 index 00000000..e69de29b diff --git a/crates/typst-kit/src/download.rs b/crates/typst-kit/src/package_downloads/http.rs similarity index 73% rename from crates/typst-kit/src/download.rs rename to crates/typst-kit/src/package_downloads/http.rs index 40084e51..3e359a55 100644 --- a/crates/typst-kit/src/download.rs +++ b/crates/typst-kit/src/package_downloads/http.rs @@ -7,27 +7,23 @@ use std::collections::VecDeque; use std::fmt::Debug; +use std::fs; use std::io::{self, ErrorKind, Read}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant}; -use ecow::EcoString; +use ecow::{eco_format, EcoString}; use native_tls::{Certificate, TlsConnector}; use once_cell::sync::OnceCell; use ureq::Response; +use typst_library::diag::{bail, PackageError, PackageResult}; +use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; +use crate::package_downloads::{DownloadState, PackageDownloader, Progress, DEFAULT_NAMESPACE}; -/// Manages progress reporting for downloads. -pub trait Progress { - /// Invoked when a download is started. - fn print_start(&mut self); +/// The default Typst registry. +pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org"; - /// Invoked repeatedly while a download is ongoing. - fn print_progress(&mut self, state: &DownloadState); - - /// Invoked when a download is finished. - fn print_finish(&mut self, state: &DownloadState); -} /// An implementation of [`Progress`] with no-op reporting, i.e., reporting /// events are swallowed. @@ -39,28 +35,14 @@ impl Progress for ProgressSink { fn print_finish(&mut self, _: &DownloadState) {} } -/// The current state of an in progress or finished download. -#[derive(Debug)] -pub struct DownloadState { - /// The expected amount of bytes to download, `None` if the response header - /// was not set. - pub content_len: Option, - /// The total amount of downloaded bytes until now. - pub total_downloaded: usize, - /// A backlog of the amount of downloaded bytes each second. - pub bytes_per_second: VecDeque, - /// The download starting instant. - pub start_time: Instant, -} - /// A minimal https client for downloading various resources. -pub struct Downloader { +pub struct HttpDownloader { user_agent: EcoString, cert_path: Option, cert: OnceCell, } -impl Downloader { +impl HttpDownloader { /// Crates a new downloader with the given user agent and no certificate. pub fn new(user_agent: impl Into) -> Self { Self { @@ -146,9 +128,28 @@ impl Downloader { let response = self.download(url)?; Ok(RemoteReader::from_response(response, progress).download()?) } + + fn parse_namespace(ns: &str) -> Result<(String, String), EcoString> { + if ns.eq(DEFAULT_NAMESPACE) { + return Ok((DEFAULT_REGISTRY.to_string(), DEFAULT_NAMESPACE.to_string())) + } + let mut parts = ns.splitn(3, ":"); + + let schema = parts.next().ok_or_else(|| { + eco_format!("expected schema in {}", ns) + })?; + let registry = parts.next().ok_or_else(|| { + eco_format!("invalid package registry in namespace {}", ns) + })?; + let ns = parts.next().ok_or_else(|| { + eco_format!("invalid package namespace in {}", ns) + })?; + + Ok((format!("{}://{}", schema, registry), ns.to_string())) + } } -impl Debug for Downloader { +impl Debug for HttpDownloader { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Downloader") .field("user_agent", &self.user_agent) @@ -257,3 +258,44 @@ impl<'p> RemoteReader<'p> { Ok(data) } } + + +impl PackageDownloader for HttpDownloader { + fn download_index(&self, spec: &VersionlessPackageSpec) -> Result, EcoString> { + let (registry, namespace) = Self::parse_namespace(spec.namespace.as_str())?; + let url = format!("{registry}/{namespace}/index.json"); + match self.download(&url) { + Ok(response) => response.into_json().map_err(|err| { + eco_format!("failed to parse package index: {err}") + }), + Err(ureq::Error::Status(404, _)) => { + bail!("failed to fetch package index (not found)") + } + Err(err) => bail!("failed to fetch package index ({err})"), + } + } + + fn download(&self, spec: &PackageSpec, package_dir: &Path, progress: &mut dyn Progress) -> PackageResult<()> { + let (registry, namespace) = Self::parse_namespace(spec.namespace.as_str()).map_err(|x| PackageError::Other(Some(x)))?; + + let url = format!( + "{}/{}/{}-{}.tar.gz", + registry, namespace, spec.name, spec.version + ); + let data = match self.download_with_progress(&url, progress) { + Ok(data) => data, + Err(ureq::Error::Status(404, _)) => { + Err(PackageError::NotFound(spec.clone()))? + } + Err(err) => { + Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))? + } + }; + + let decompressed = flate2::read::GzDecoder::new(data.as_slice()); + tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| { + fs::remove_dir_all(package_dir).ok(); + PackageError::MalformedArchive(Some(eco_format!("{err}"))) + }) + } +} \ No newline at end of file diff --git a/crates/typst-kit/src/package_downloads/mod.rs b/crates/typst-kit/src/package_downloads/mod.rs new file mode 100644 index 00000000..2e2a4717 --- /dev/null +++ b/crates/typst-kit/src/package_downloads/mod.rs @@ -0,0 +1,121 @@ +use std::collections::VecDeque; +use std::fmt::Debug; +use std::path::{Path, PathBuf}; +use std::time::Instant; +use ecow::{eco_format, EcoString}; +use typst_library::diag::{PackageError, PackageResult}; +use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; + +/// The public namespace in the default Typst registry. +pub const DEFAULT_NAMESPACE: &str = "preview"; + +#[cfg(feature = "downloads_http")] +mod http; + +#[cfg(feature = "downloads_git")] +mod git; + +pub trait PackageDownloader : Debug + Sync + Send { + fn download_index(&self, spec: &VersionlessPackageSpec) -> Result, EcoString>; + + fn download(&self, spec: &PackageSpec, package_dir: &Path, progress: &mut dyn Progress) -> PackageResult<()>; +} + +/// The current state of an in progress or finished download. +#[derive(Debug)] +pub struct DownloadState { + /// The expected amount of bytes to download, `None` if the response header + /// was not set. + pub content_len: Option, + /// The total amount of downloaded bytes until now. + pub total_downloaded: usize, + /// A backlog of the amount of downloaded bytes each second. + pub bytes_per_second: VecDeque, + /// The download starting instant. + pub start_time: Instant, +} + +/// Manages progress reporting for downloads. +pub trait Progress { + /// Invoked when a download is started. + fn print_start(&mut self); + + /// Invoked repeatedly while a download is ongoing. + fn print_progress(&mut self, state: &DownloadState); + + /// Invoked when a download is finished. + fn print_finish(&mut self, state: &DownloadState); +} + +#[derive(Debug)] +pub struct Downloader{ + http_downloader: Option>, + git_downloader: Option>, +} + +impl Downloader { + pub fn new(cert: Option) -> Self { + Self { + http_downloader: Self::make_http_downloader(cert.clone()), + git_downloader: Self::make_git_downloader(cert), + } + } + + fn make_http_downloader(cert: Option) -> Option>{ + #[cfg(not(feature = "downloads_http"))] + { None } + + #[cfg(feature = "downloads_http")] + { + let user_agent = concat!("typst/", env!("CARGO_PKG_VERSION")); + match cert { + Some(cert_path) => Some(Box::new(http::HttpDownloader::with_path(user_agent, cert_path))), + None => Some(Box::new(http::HttpDownloader::new(user_agent))), + } + } + } + + fn make_git_downloader(_cert: Option) -> Option>{ + #[cfg(not(feature = "downloads_http"))] + { None } + + #[cfg(feature = "downloads_http")] + { + None + } + } + + fn get_downloader(&self, ns: &str) -> Result<&Box, PackageError> { + let download_type = ns.splitn(2, ":").next(); + + match download_type { + #[cfg(feature = "downloads_http")] + Some("http") => self.http_downloader.as_ref().ok_or_else(|| PackageError::Other(Some(EcoString::from("Http downloader has not been initialized correctly")))), + #[cfg(feature = "downloads_http")] + Some("https") => self.http_downloader.as_ref().ok_or_else(|| PackageError::Other(Some(EcoString::from("Https downloader has not been initialized correctly")))), + #[cfg(feature = "downloads_http")] + Some("preview") => self.http_downloader.as_ref().ok_or_else(|| PackageError::Other(Some(EcoString::from("Https downloader has not been initialized correctly")))), + + #[cfg(feature = "downloads_git")] + Some("git") => self.git_downloader.as_ref().ok_or_else(|| PackageError::Other(Some(EcoString::from("Git downloader has not been initialized correctly")))), + + Some(dwld) => Err(PackageError::Other(Some(eco_format!("Unknown downloader type: {}", dwld)))), + None => Err(PackageError::Other(Some(EcoString::from("No downloader type specified")))), + } + } +} + + +impl PackageDownloader for Downloader { + fn download_index(&self, spec: &VersionlessPackageSpec) -> Result, EcoString> { + let downloader = self.get_downloader(spec.namespace.as_str())?; + downloader.download_index(spec) + } + + fn download(&self, spec: &PackageSpec, package_dir: &Path, progress: &mut dyn Progress) -> PackageResult<()> { + let downloader = self.get_downloader(spec.namespace.as_str())?; + downloader.download(spec, package_dir, progress) + } +} + + diff --git a/crates/typst-syntax/src/package.rs b/crates/typst-syntax/src/package.rs index 387057f3..8a0407aa 100644 --- a/crates/typst-syntax/src/package.rs +++ b/crates/typst-syntax/src/package.rs @@ -263,15 +263,36 @@ impl Display for VersionlessPackageSpec { } } +fn is_namespace_valid(namespace: &str) -> bool { + if is_ident(namespace){ + //standard namespace + return true + } + + //if not ident, the namespace should be formed as @: + let mut tokenized = namespace.splitn(2, ":"); + + //package type + if tokenized.next().is_none_or(|x| !is_ident(x)) { + return false + } + + //the package_path parsing is left to the downloader implementation + true +} + fn parse_namespace<'s>(s: &mut Scanner<'s>) -> Result<&'s str, EcoString> { if !s.eat_if('@') { Err("package specification must start with '@'")?; } + //todo: allow for multiple slashes in the by eating until last slash let namespace = s.eat_until('/'); if namespace.is_empty() { Err("package specification is missing namespace")?; - } else if !is_ident(namespace) { + } + + if !is_namespace_valid(namespace) { Err(eco_format!("`{namespace}` is not a valid package namespace"))?; } From 3436f825f25d5a1c6e33257d47b358c056be5ab7 Mon Sep 17 00:00:00 2001 From: Stefano Fontana Date: Sat, 14 Dec 2024 17:59:55 +0100 Subject: [PATCH 2/8] git download method --- Cargo.lock | 100 +++++++++++++++ Cargo.toml | 2 + crates/typst-kit/Cargo.toml | 4 +- crates/typst-kit/src/package_downloads/git.rs | 117 ++++++++++++++++++ .../typst-kit/src/package_downloads/http.rs | 10 +- crates/typst-kit/src/package_downloads/mod.rs | 7 +- 6 files changed, 233 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94ce026e..7e35b335 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "auth-git2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3810b5af212b013fe7302b12d86616c6c39a48e18f2e4b812a5a9e5710213791" +dependencies = [ + "dirs", + "git2", + "terminal-prompt", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -863,6 +874,21 @@ dependencies = [ "weezl", ] +[[package]] +name = "git2" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" +dependencies = [ + "bitflags 2.6.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "half" version = "2.4.1" @@ -1322,6 +1348,20 @@ dependencies = [ "cc", ] +[[package]] +name = "libgit2-sys" +version = "0.17.0+1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.8" @@ -1339,6 +1379,32 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2505,6 +2571,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal-prompt" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572818b3472910acbd5dff46a3413715c18e934b071ab2ba464a7b2c2af16376" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "terminal_size" version = "0.4.0" @@ -2854,11 +2930,13 @@ dependencies = [ name = "typst-kit" version = "0.12.0" dependencies = [ + "auth-git2", "dirs", "ecow", "env_proxy", "flate2", "fontdb", + "git2", "native-tls", "once_cell", "openssl", @@ -3422,6 +3500,22 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.9" @@ -3431,6 +3525,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index b20d54e8..f6605b0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ typst-timing = { path = "crates/typst-timing", version = "0.12.0" } typst-utils = { path = "crates/typst-utils", version = "0.12.0" } typst-assets = { git = "https://github.com/typst/typst-assets", rev = "8cccef9" } typst-dev-assets = { git = "https://github.com/typst/typst-dev-assets", rev = "b07d156" } +auth-git2 = "0.5.5" arrayvec = "0.7.4" az = "1.2" base64 = "0.22" @@ -58,6 +59,7 @@ env_proxy = "0.4" flate2 = "1" fontdb = { version = "0.21", default-features = false } fs_extra = "1.3" +git2 = "0.19.0" hayagriva = "0.8" heck = "0.5" hypher = "0.1.4" diff --git a/crates/typst-kit/Cargo.toml b/crates/typst-kit/Cargo.toml index 666b0dc1..94f3fa40 100644 --- a/crates/typst-kit/Cargo.toml +++ b/crates/typst-kit/Cargo.toml @@ -25,6 +25,8 @@ native-tls = { workspace = true, optional = true } once_cell = { workspace = true } tar = { workspace = true, optional = true } ureq = { workspace = true, optional = true } +git2 = { workspace = true, optional = true } +auth-git2 = { workspace = true, optional = true } # Explicitly depend on OpenSSL if applicable, so that we can add the # `openssl/vendored` feature to it if `vendor-openssl` is enabled. @@ -40,7 +42,7 @@ fonts = ["dep:fontdb", "fontdb/memmap", "fontdb/fontconfig"] # Add generic downloading utilities downloads = ["downloads_http", "downloads_git"] downloads_http = ["dep:env_proxy", "dep:native-tls", "dep:ureq", "dep:openssl"] -downloads_git = [] +downloads_git = ["git2", "auth-git2"] # Add package downloading utilities, implies `downloads` packages = ["downloads", "dep:dirs", "dep:flate2", "dep:tar"] diff --git a/crates/typst-kit/src/package_downloads/git.rs b/crates/typst-kit/src/package_downloads/git.rs index e69de29b..6999eebf 100644 --- a/crates/typst-kit/src/package_downloads/git.rs +++ b/crates/typst-kit/src/package_downloads/git.rs @@ -0,0 +1,117 @@ +use std::cell::{RefCell, RefMut}; +use std::collections::VecDeque; +use std::fmt::Debug; +use std::fs; +use std::io::{self, ErrorKind, Read}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use auth_git2::GitAuthenticator; +use crate::package_downloads::{ + DownloadState, PackageDownloader, Progress, DEFAULT_NAMESPACE, +}; +use ecow::{eco_format, EcoString}; +use native_tls::{Certificate, TlsConnector}; +use once_cell::sync::OnceCell; +use typst_library::diag::{bail, PackageError, PackageResult}; +use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; +use git2::{AutotagOption, FetchOptions, Progress as GitProgress, RemoteCallbacks}; +use git2::build::{CheckoutBuilder, RepoBuilder}; +use typst_library::html::tag::form; + +#[derive(Debug)] +pub struct GitDownloader; + +impl GitDownloader { + pub fn new() -> Self { + Self {} + } + + pub fn download_with_progress( + &self, + repo: &str, + tag: &str, + dest: &Path, + progress: &mut dyn Progress, + ) -> Result<(), EcoString> { + progress.print_start(); + + eprintln!("{} {} {}", repo, tag, dest.display()); + + let state = DownloadState{ + content_len: None, + total_downloaded: 0, + bytes_per_second: VecDeque::from(vec![0; 5]), + start_time: Instant::now(), + }; + + + let auth = GitAuthenticator::default(); + let git_config = git2::Config::open_default().map_err(|err| {EcoString::from(format!("{:?}", err))})?; + + let mut fetch_options = FetchOptions::new(); + let mut remote_callbacks = RemoteCallbacks::new(); + + remote_callbacks.credentials(auth.credentials(&git_config)); + fetch_options + .remote_callbacks(remote_callbacks); + + let repo = RepoBuilder::new() + .fetch_options(fetch_options) + .clone(repo, dest).map_err(|err| {EcoString::from(format!("{:?}", err))})?; + + let (object, reference) = repo + .revparse_ext(tag).map_err(|err| {EcoString::from(format!("{:?}", err))})?; + repo.checkout_tree(&object, None).map_err(|err| {EcoString::from(format!("{:?}", err))})?; + + match reference { + // gref is an actual reference like branches or tags + Some(gref) => repo.set_head(gref.name().unwrap()), + // this is a commit, not a reference + None => repo.set_head_detached(object.id()), + }.map_err(|err| {EcoString::from(format!("{:?}", err))})?; + + progress.print_finish(&state); + Ok(()) + } + + fn parse_namespace(ns: &str, name: &str) -> Result { + + let mut parts = ns.splitn(2, ":"); + let schema = parts.next().ok_or_else(|| { + eco_format!("expected schema in {}", ns) + })?; + let repo = parts.next().ok_or_else(|| { + eco_format!("invalid package repo {}", ns) + })?; + + if !schema.eq("git") { + Err(eco_format!("invalid schema in {}", ns))? + } + + Ok(format!("{}/{}.git", repo, name)) + } +} + +impl PackageDownloader for GitDownloader { + fn download_index( + &self, + spec: &VersionlessPackageSpec, + ) -> Result, EcoString> { + + //todo ls-remote + todo!() + } + + fn download( + &self, + spec: &PackageSpec, + package_dir: &Path, + progress: &mut dyn Progress, + ) -> PackageResult<()> { + let repo = Self::parse_namespace(spec.namespace.as_str(), spec.name.as_str()).map_err(|x| PackageError::Other(Some(x)))?; + let tag = format!("v{}", spec.version); + self.download_with_progress(repo.as_str(), tag.as_str(), package_dir, progress).map_err(|x| PackageError::Other(Some(x))) + } +} diff --git a/crates/typst-kit/src/package_downloads/http.rs b/crates/typst-kit/src/package_downloads/http.rs index 3e359a55..d446508b 100644 --- a/crates/typst-kit/src/package_downloads/http.rs +++ b/crates/typst-kit/src/package_downloads/http.rs @@ -89,7 +89,7 @@ impl HttpDownloader { /// Download binary data from the given url. #[allow(clippy::result_large_err)] - pub fn download(&self, url: &str) -> Result { + pub fn perform_download(&self, url: &str) -> Result { let mut builder = ureq::AgentBuilder::new(); let mut tls = TlsConnector::builder(); @@ -125,7 +125,7 @@ impl HttpDownloader { progress: &mut dyn Progress, ) -> Result, ureq::Error> { progress.print_start(); - let response = self.download(url)?; + let response = self.perform_download(url)?; Ok(RemoteReader::from_response(response, progress).download()?) } @@ -145,6 +145,10 @@ impl HttpDownloader { eco_format!("invalid package namespace in {}", ns) })?; + if !schema.eq("http") && !schema.eq("https") { + Err(eco_format!("invalid schema in {}", ns))? + } + Ok((format!("{}://{}", schema, registry), ns.to_string())) } } @@ -264,7 +268,7 @@ impl PackageDownloader for HttpDownloader { fn download_index(&self, spec: &VersionlessPackageSpec) -> Result, EcoString> { let (registry, namespace) = Self::parse_namespace(spec.namespace.as_str())?; let url = format!("{registry}/{namespace}/index.json"); - match self.download(&url) { + match self.perform_download(&url) { Ok(response) => response.into_json().map_err(|err| { eco_format!("failed to parse package index: {err}") }), diff --git a/crates/typst-kit/src/package_downloads/mod.rs b/crates/typst-kit/src/package_downloads/mod.rs index 2e2a4717..cf7ef7a1 100644 --- a/crates/typst-kit/src/package_downloads/mod.rs +++ b/crates/typst-kit/src/package_downloads/mod.rs @@ -5,6 +5,7 @@ use std::time::Instant; use ecow::{eco_format, EcoString}; use typst_library::diag::{PackageError, PackageResult}; use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; +use crate::package_downloads::git::GitDownloader; /// The public namespace in the default Typst registry. pub const DEFAULT_NAMESPACE: &str = "preview"; @@ -76,12 +77,12 @@ impl Downloader { } fn make_git_downloader(_cert: Option) -> Option>{ - #[cfg(not(feature = "downloads_http"))] + #[cfg(not(feature = "downloads_git"))] { None } - #[cfg(feature = "downloads_http")] + #[cfg(feature = "downloads_git")] { - None + Some(Box::new(GitDownloader::new())) } } From 0a1df1ee67ad1c158416553cfc2781ac595fd65f Mon Sep 17 00:00:00 2001 From: Stefano Fontana Date: Sat, 14 Dec 2024 18:05:35 +0100 Subject: [PATCH 3/8] fixed warinings --- crates/typst-kit/src/package_downloads/git.rs | 26 ++++++------------- .../typst-kit/src/package_downloads/http.rs | 9 ------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/crates/typst-kit/src/package_downloads/git.rs b/crates/typst-kit/src/package_downloads/git.rs index 6999eebf..d92544e7 100644 --- a/crates/typst-kit/src/package_downloads/git.rs +++ b/crates/typst-kit/src/package_downloads/git.rs @@ -1,24 +1,16 @@ -use std::cell::{RefCell, RefMut}; use std::collections::VecDeque; use std::fmt::Debug; -use std::fs; -use std::io::{self, ErrorKind, Read}; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::path::Path; +use std::time::Instant; use auth_git2::GitAuthenticator; use crate::package_downloads::{ - DownloadState, PackageDownloader, Progress, DEFAULT_NAMESPACE, + DownloadState, PackageDownloader, Progress, }; use ecow::{eco_format, EcoString}; -use native_tls::{Certificate, TlsConnector}; -use once_cell::sync::OnceCell; -use typst_library::diag::{bail, PackageError, PackageResult}; +use typst_library::diag::{PackageError, PackageResult}; use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; -use git2::{AutotagOption, FetchOptions, Progress as GitProgress, RemoteCallbacks}; -use git2::build::{CheckoutBuilder, RepoBuilder}; -use typst_library::html::tag::form; +use git2::{FetchOptions, RemoteCallbacks}; +use git2::build::RepoBuilder; #[derive(Debug)] pub struct GitDownloader; @@ -97,11 +89,9 @@ impl GitDownloader { impl PackageDownloader for GitDownloader { fn download_index( &self, - spec: &VersionlessPackageSpec, + _spec: &VersionlessPackageSpec, ) -> Result, EcoString> { - - //todo ls-remote - todo!() + Err(eco_format!("Downloading index is not supported for git repositories")) } fn download( diff --git a/crates/typst-kit/src/package_downloads/http.rs b/crates/typst-kit/src/package_downloads/http.rs index d446508b..a1cb71df 100644 --- a/crates/typst-kit/src/package_downloads/http.rs +++ b/crates/typst-kit/src/package_downloads/http.rs @@ -63,15 +63,6 @@ impl HttpDownloader { } } - /// Crates a new downloader with the given user agent and certificate. - pub fn with_cert(user_agent: impl Into, cert: Certificate) -> Self { - Self { - user_agent: user_agent.into(), - cert_path: None, - cert: OnceCell::with_value(cert), - } - } - /// Returns the certificate this client is using, if a custom certificate /// is used it is loaded on first access. /// From b5689cfc724980e4693affe58ffd0348e794ef98 Mon Sep 17 00:00:00 2001 From: Stefano Fontana Date: Sat, 14 Dec 2024 18:23:08 +0100 Subject: [PATCH 4/8] documentation --- crates/typst-kit/src/package_downloads/git.rs | 15 +++++++++++++ .../typst-kit/src/package_downloads/http.rs | 13 +++++++++++ crates/typst-kit/src/package_downloads/mod.rs | 22 +++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/crates/typst-kit/src/package_downloads/git.rs b/crates/typst-kit/src/package_downloads/git.rs index d92544e7..5ca3a5c4 100644 --- a/crates/typst-kit/src/package_downloads/git.rs +++ b/crates/typst-kit/src/package_downloads/git.rs @@ -68,6 +68,21 @@ impl GitDownloader { Ok(()) } + /// Parses the namespace of the package into the correct registry and namespace. + /// The namespace format is the following: + /// + /// @git: + /// + /// The final repository cloned will be formed by the git host and the repository name + /// with the adequate extension, checking out to the tag specified by the version in the format + /// v.. + /// + /// For example, the package + /// @git:git@github.com:typst/package:0.1.0 + /// will result in the cloning of the repository git@github.com:typst/package.git + /// and the checkout and detached head state at tag v0.1.0 + /// + /// NOTE: no index download is possible. fn parse_namespace(ns: &str, name: &str) -> Result { let mut parts = ns.splitn(2, ":"); diff --git a/crates/typst-kit/src/package_downloads/http.rs b/crates/typst-kit/src/package_downloads/http.rs index a1cb71df..413698f9 100644 --- a/crates/typst-kit/src/package_downloads/http.rs +++ b/crates/typst-kit/src/package_downloads/http.rs @@ -120,6 +120,19 @@ impl HttpDownloader { Ok(RemoteReader::from_response(response, progress).download()?) } + /// Parses the namespace of the package into the correct registry and namespace. + /// The namespace format is the following: + /// + /// @http[s]::/package-name>:package-version + /// + /// resulting in the package location to be resolved as + /// http[s]:////-.tar.gz + /// + /// and the index to be resolved as + /// http[s]:////index.json + /// + /// NOTE: preview namespace is treated as the namespace formed as + /// @https:packages.typst.org:preview/package-name>:package-version fn parse_namespace(ns: &str) -> Result<(String, String), EcoString> { if ns.eq(DEFAULT_NAMESPACE) { return Ok((DEFAULT_REGISTRY.to_string(), DEFAULT_NAMESPACE.to_string())) diff --git a/crates/typst-kit/src/package_downloads/mod.rs b/crates/typst-kit/src/package_downloads/mod.rs index cf7ef7a1..56c87591 100644 --- a/crates/typst-kit/src/package_downloads/mod.rs +++ b/crates/typst-kit/src/package_downloads/mod.rs @@ -10,15 +10,23 @@ use crate::package_downloads::git::GitDownloader; /// The public namespace in the default Typst registry. pub const DEFAULT_NAMESPACE: &str = "preview"; +/*========BEGIN DOWNLOAD METHODS DECLARATION=========*/ #[cfg(feature = "downloads_http")] mod http; #[cfg(feature = "downloads_git")] mod git; +/*========END DOWNLOAD METHODS DECLARATION===========*/ +/// Trait abstraction for package a downloader. pub trait PackageDownloader : Debug + Sync + Send { + + /// Download the repository index and returns the + /// list of PackageInfo elements contained in it. fn download_index(&self, spec: &VersionlessPackageSpec) -> Result, EcoString>; + /// Download a package from a remote repository/registry + /// and writes it in the file system cache directory fn download(&self, spec: &PackageSpec, package_dir: &Path, progress: &mut dyn Progress) -> PackageResult<()>; } @@ -48,13 +56,18 @@ pub trait Progress { fn print_finish(&mut self, state: &DownloadState); } +/// The downloader object used for downloading packages #[derive(Debug)] pub struct Downloader{ + ///List of all available downloaders which can be instantiated at runtime http_downloader: Option>, git_downloader: Option>, } + impl Downloader { + /// Construct the Downloader object instantiating all the available methods. + /// The methods can be compile-time selected by features. pub fn new(cert: Option) -> Self { Self { http_downloader: Self::make_http_downloader(cert.clone()), @@ -62,6 +75,7 @@ impl Downloader { } } + /// Creation function for the HTTP(S) download method fn make_http_downloader(cert: Option) -> Option>{ #[cfg(not(feature = "downloads_http"))] { None } @@ -76,6 +90,7 @@ impl Downloader { } } + /// Creation function for the GIT clone method fn make_git_downloader(_cert: Option) -> Option>{ #[cfg(not(feature = "downloads_git"))] { None } @@ -86,6 +101,13 @@ impl Downloader { } } + /// Returns the correct downloader in function of the package namespace. + /// The remote location of a package is encoded in its namespace in the form + /// @: + /// + /// It's the downloader instance's job to parse the source path in any substructure. + /// + /// NOTE: Treating @preview as a special case of the https downloader. fn get_downloader(&self, ns: &str) -> Result<&Box, PackageError> { let download_type = ns.splitn(2, ":").next(); From d687d23e5e05f776dc5339ed8d282af9c25c863a Mon Sep 17 00:00:00 2001 From: Stefano Fontana Date: Sun, 15 Dec 2024 12:35:06 +0100 Subject: [PATCH 5/8] cli updater adaptation and clippy fixes --- crates/typst-cli/Cargo.toml | 2 +- crates/typst-cli/src/update.rs | 13 +-- crates/typst-kit/src/lib.rs | 4 +- crates/typst-kit/src/package.rs | 15 +-- crates/typst-kit/src/package_downloads/git.rs | 57 +++++----- .../typst-kit/src/package_downloads/http.rs | 66 +++++++----- crates/typst-kit/src/package_downloads/mod.rs | 101 ++++++++++++------ crates/typst-syntax/src/package.rs | 9 +- 8 files changed, 159 insertions(+), 108 deletions(-) diff --git a/crates/typst-cli/Cargo.toml b/crates/typst-cli/Cargo.toml index 7e9b93f9..b29e4fb7 100644 --- a/crates/typst-cli/Cargo.toml +++ b/crates/typst-cli/Cargo.toml @@ -21,7 +21,7 @@ doc = false typst = { workspace = true } typst-eval = { workspace = true } typst-html = { workspace = true } -typst-kit = { workspace = true } +typst-kit = { workspace = true, features = ["downloads_http"] } typst-macros = { workspace = true } typst-pdf = { workspace = true } typst-render = { workspace = true } diff --git a/crates/typst-cli/src/update.rs b/crates/typst-cli/src/update.rs index ec8ca71e..049e79e4 100644 --- a/crates/typst-cli/src/update.rs +++ b/crates/typst-cli/src/update.rs @@ -7,12 +7,12 @@ use semver::Version; use serde::Deserialize; use tempfile::NamedTempFile; use typst::diag::{bail, StrResult}; -use typst_kit::download::Downloader; +use typst_kit::package_downloads::http::HttpDownloader; use xz2::bufread::XzDecoder; use zip::ZipArchive; use crate::args::UpdateCommand; -use crate::download::{self, PrintDownload}; +use crate::download::PrintDownload; const TYPST_GITHUB_ORG: &str = "typst"; const TYPST_REPO: &str = "typst"; @@ -91,7 +91,8 @@ pub fn update(command: &UpdateCommand) -> StrResult<()> { fs::copy(current_exe, &backup_path) .map_err(|err| eco_format!("failed to create backup ({err})"))?; - let downloader = download::downloader(); + //no certificate is needed to download from GitHub + let downloader = HttpDownloader::new(HttpDownloader::default_user_agent()); let release = Release::from_tag(command.version.as_ref(), &downloader)?; if !update_needed(&release)? && !command.force { @@ -133,7 +134,7 @@ impl Release { /// Typst repository. pub fn from_tag( tag: Option<&Version>, - downloader: &Downloader, + downloader: &HttpDownloader, ) -> StrResult { let url = match tag { Some(tag) => format!( @@ -144,7 +145,7 @@ impl Release { ), }; - match downloader.download(&url) { + match downloader.perform_download(&url) { Ok(response) => response.into_json().map_err(|err| { eco_format!("failed to parse release information ({err})") }), @@ -161,7 +162,7 @@ impl Release { pub fn download_binary( &self, asset_name: &str, - downloader: &Downloader, + downloader: &HttpDownloader, ) -> StrResult> { let asset = self.assets.iter().find(|a| a.name.starts_with(asset_name)).ok_or( eco_format!( diff --git a/crates/typst-kit/src/lib.rs b/crates/typst-kit/src/lib.rs index 6c2c3e5b..6b6fcf10 100644 --- a/crates/typst-kit/src/lib.rs +++ b/crates/typst-kit/src/lib.rs @@ -19,9 +19,9 @@ //! [download]. It is enabled by the `packages` feature flag and implies the //! `downloads` feature flag. -#[cfg(feature = "downloads")] -pub mod package_downloads; #[cfg(feature = "fonts")] pub mod fonts; #[cfg(feature = "packages")] pub mod package; +#[cfg(feature = "downloads")] +pub mod package_downloads; diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs index 440dbef9..1708cada 100644 --- a/crates/typst-kit/src/package.rs +++ b/crates/typst-kit/src/package.rs @@ -1,13 +1,13 @@ //! Download and unpack packages and package indices. use std::path::{Path, PathBuf}; +use crate::package_downloads::{Downloader, PackageDownloader, Progress}; use ecow::eco_format; use once_cell::sync::OnceCell; use typst_library::diag::{PackageError, PackageResult, StrResult}; use typst_syntax::package::{ PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec, }; -use crate::package_downloads::{Downloader, PackageDownloader, Progress}; /// The default packages sub directory within the package and package cache paths. pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages"; @@ -97,12 +97,12 @@ impl PackageStorage { &self, spec: &VersionlessPackageSpec, ) -> StrResult { - // Same logical flow as per package download. Check package path, then check online. // Do not check in the data directory because the latter is not intended for storage // of local packages. let subdir = format!("{}/{}", spec.namespace, spec.name); - let res = self.package_path + let res = self + .package_path .iter() .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok()) .flatten() @@ -124,7 +124,10 @@ impl PackageStorage { } /// Download the package index. The result of this is cached for efficiency. - pub fn download_index(&self, spec: &VersionlessPackageSpec) -> StrResult<&[PackageInfo]> { + pub fn download_index( + &self, + spec: &VersionlessPackageSpec, + ) -> StrResult<&[PackageInfo]> { self.index .get_or_try_init(|| self.downloader.download_index(spec)) .map(AsRef::as_ref) @@ -147,8 +150,8 @@ impl PackageStorage { } else { Err(PackageError::NotFound(spec.clone())) } - }, - val => val + } + val => val, } } } diff --git a/crates/typst-kit/src/package_downloads/git.rs b/crates/typst-kit/src/package_downloads/git.rs index 5ca3a5c4..d8005c65 100644 --- a/crates/typst-kit/src/package_downloads/git.rs +++ b/crates/typst-kit/src/package_downloads/git.rs @@ -1,16 +1,14 @@ +use crate::package_downloads::{DownloadState, PackageDownloader, Progress}; +use auth_git2::GitAuthenticator; +use ecow::{eco_format, EcoString}; +use git2::build::RepoBuilder; +use git2::{FetchOptions, RemoteCallbacks}; use std::collections::VecDeque; use std::fmt::Debug; use std::path::Path; use std::time::Instant; -use auth_git2::GitAuthenticator; -use crate::package_downloads::{ - DownloadState, PackageDownloader, Progress, -}; -use ecow::{eco_format, EcoString}; use typst_library::diag::{PackageError, PackageResult}; use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; -use git2::{FetchOptions, RemoteCallbacks}; -use git2::build::RepoBuilder; #[derive(Debug)] pub struct GitDownloader; @@ -31,38 +29,41 @@ impl GitDownloader { eprintln!("{} {} {}", repo, tag, dest.display()); - let state = DownloadState{ + let state = DownloadState { content_len: None, total_downloaded: 0, bytes_per_second: VecDeque::from(vec![0; 5]), start_time: Instant::now(), }; - let auth = GitAuthenticator::default(); - let git_config = git2::Config::open_default().map_err(|err| {EcoString::from(format!("{:?}", err))})?; + let git_config = git2::Config::open_default() + .map_err(|err| EcoString::from(format!("{err}")))?; let mut fetch_options = FetchOptions::new(); let mut remote_callbacks = RemoteCallbacks::new(); remote_callbacks.credentials(auth.credentials(&git_config)); - fetch_options - .remote_callbacks(remote_callbacks); + fetch_options.remote_callbacks(remote_callbacks); let repo = RepoBuilder::new() .fetch_options(fetch_options) - .clone(repo, dest).map_err(|err| {EcoString::from(format!("{:?}", err))})?; + .clone(repo, dest) + .map_err(|err| EcoString::from(format!("{err}")))?; let (object, reference) = repo - .revparse_ext(tag).map_err(|err| {EcoString::from(format!("{:?}", err))})?; - repo.checkout_tree(&object, None).map_err(|err| {EcoString::from(format!("{:?}", err))})?; + .revparse_ext(tag) + .map_err(|err| EcoString::from(format!("{err}")))?; + repo.checkout_tree(&object, None) + .map_err(|err| EcoString::from(format!("{err}")))?; match reference { // gref is an actual reference like branches or tags Some(gref) => repo.set_head(gref.name().unwrap()), // this is a commit, not a reference None => repo.set_head_detached(object.id()), - }.map_err(|err| {EcoString::from(format!("{:?}", err))})?; + } + .map_err(|err| EcoString::from(format!("{err}")))?; progress.print_finish(&state); Ok(()) @@ -78,33 +79,31 @@ impl GitDownloader { /// v.. /// /// For example, the package - /// @git:git@github.com:typst/package:0.1.0 + /// @git:git@github.com:typst/package:0.0 /// will result in the cloning of the repository git@github.com:typst/package.git /// and the checkout and detached head state at tag v0.1.0 /// /// NOTE: no index download is possible. fn parse_namespace(ns: &str, name: &str) -> Result { - let mut parts = ns.splitn(2, ":"); - let schema = parts.next().ok_or_else(|| { - eco_format!("expected schema in {}", ns) - })?; - let repo = parts.next().ok_or_else(|| { - eco_format!("invalid package repo {}", ns) - })?; + let schema = + parts.next().ok_or_else(|| eco_format!("expected schema in {}", ns))?; + let repo = parts + .next() + .ok_or_else(|| eco_format!("invalid package repo {}", ns))?; if !schema.eq("git") { Err(eco_format!("invalid schema in {}", ns))? } - Ok(format!("{}/{}.git", repo, name)) + Ok(format!("{repo}/{name}.git")) } } impl PackageDownloader for GitDownloader { fn download_index( &self, - _spec: &VersionlessPackageSpec, + _spec: &VersionlessPackageSpec, ) -> Result, EcoString> { Err(eco_format!("Downloading index is not supported for git repositories")) } @@ -115,8 +114,10 @@ impl PackageDownloader for GitDownloader { package_dir: &Path, progress: &mut dyn Progress, ) -> PackageResult<()> { - let repo = Self::parse_namespace(spec.namespace.as_str(), spec.name.as_str()).map_err(|x| PackageError::Other(Some(x)))?; + let repo = Self::parse_namespace(spec.namespace.as_str(), spec.name.as_str()) + .map_err(|x| PackageError::Other(Some(x)))?; let tag = format!("v{}", spec.version); - self.download_with_progress(repo.as_str(), tag.as_str(), package_dir, progress).map_err(|x| PackageError::Other(Some(x))) + self.download_with_progress(repo.as_str(), tag.as_str(), package_dir, progress) + .map_err(|x| PackageError::Other(Some(x))) } } diff --git a/crates/typst-kit/src/package_downloads/http.rs b/crates/typst-kit/src/package_downloads/http.rs index 413698f9..f00bb1ab 100644 --- a/crates/typst-kit/src/package_downloads/http.rs +++ b/crates/typst-kit/src/package_downloads/http.rs @@ -13,18 +13,19 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant}; +use crate::package_downloads::{ + DownloadState, PackageDownloader, Progress, DEFAULT_NAMESPACE, +}; use ecow::{eco_format, EcoString}; use native_tls::{Certificate, TlsConnector}; use once_cell::sync::OnceCell; -use ureq::Response; use typst_library::diag::{bail, PackageError, PackageResult}; use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; -use crate::package_downloads::{DownloadState, PackageDownloader, Progress, DEFAULT_NAMESPACE}; +use ureq::Response; /// The default Typst registry. pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org"; - /// An implementation of [`Progress`] with no-op reporting, i.e., reporting /// events are swallowed. pub struct ProgressSink; @@ -43,6 +44,10 @@ pub struct HttpDownloader { } impl HttpDownloader { + pub fn default_user_agent() -> String { + format!("typst-kit/{}", env!("CARGO_PKG_VERSION")) + } + /// Crates a new downloader with the given user agent and no certificate. pub fn new(user_agent: impl Into) -> Self { Self { @@ -135,25 +140,24 @@ impl HttpDownloader { /// @https:packages.typst.org:preview/package-name>:package-version fn parse_namespace(ns: &str) -> Result<(String, String), EcoString> { if ns.eq(DEFAULT_NAMESPACE) { - return Ok((DEFAULT_REGISTRY.to_string(), DEFAULT_NAMESPACE.to_string())) + return Ok((DEFAULT_REGISTRY.to_string(), DEFAULT_NAMESPACE.to_string())); } let mut parts = ns.splitn(3, ":"); - let schema = parts.next().ok_or_else(|| { - eco_format!("expected schema in {}", ns) - })?; - let registry = parts.next().ok_or_else(|| { - eco_format!("invalid package registry in namespace {}", ns) - })?; - let ns = parts.next().ok_or_else(|| { - eco_format!("invalid package namespace in {}", ns) - })?; + let schema = + parts.next().ok_or_else(|| eco_format!("expected schema in {}", ns))?; + let registry = parts + .next() + .ok_or_else(|| eco_format!("invalid package registry in namespace {}", ns))?; + let ns = parts + .next() + .ok_or_else(|| eco_format!("invalid package namespace in {}", ns))?; if !schema.eq("http") && !schema.eq("https") { Err(eco_format!("invalid schema in {}", ns))? } - Ok((format!("{}://{}", schema, registry), ns.to_string())) + Ok((format!("{schema}://{registry}"), ns.to_string())) } } @@ -267,15 +271,17 @@ impl<'p> RemoteReader<'p> { } } - impl PackageDownloader for HttpDownloader { - fn download_index(&self, spec: &VersionlessPackageSpec) -> Result, EcoString> { + fn download_index( + &self, + spec: &VersionlessPackageSpec, + ) -> Result, EcoString> { let (registry, namespace) = Self::parse_namespace(spec.namespace.as_str())?; let url = format!("{registry}/{namespace}/index.json"); match self.perform_download(&url) { - Ok(response) => response.into_json().map_err(|err| { - eco_format!("failed to parse package index: {err}") - }), + Ok(response) => response + .into_json() + .map_err(|err| eco_format!("failed to parse package index: {err}")), Err(ureq::Error::Status(404, _)) => { bail!("failed to fetch package index (not found)") } @@ -283,21 +289,23 @@ impl PackageDownloader for HttpDownloader { } } - fn download(&self, spec: &PackageSpec, package_dir: &Path, progress: &mut dyn Progress) -> PackageResult<()> { - let (registry, namespace) = Self::parse_namespace(spec.namespace.as_str()).map_err(|x| PackageError::Other(Some(x)))?; + fn download( + &self, + spec: &PackageSpec, + package_dir: &Path, + progress: &mut dyn Progress, + ) -> PackageResult<()> { + let (registry, namespace) = Self::parse_namespace(spec.namespace.as_str()) + .map_err(|x| PackageError::Other(Some(x)))?; - let url = format!( - "{}/{}/{}-{}.tar.gz", - registry, namespace, spec.name, spec.version - ); + let url = + format!("{}/{}/{}-{}.tar.gz", registry, namespace, spec.name, spec.version); let data = match self.download_with_progress(&url, progress) { Ok(data) => data, Err(ureq::Error::Status(404, _)) => { Err(PackageError::NotFound(spec.clone()))? } - Err(err) => { - Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))? - } + Err(err) => Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))?, }; let decompressed = flate2::read::GzDecoder::new(data.as_slice()); @@ -306,4 +314,4 @@ impl PackageDownloader for HttpDownloader { PackageError::MalformedArchive(Some(eco_format!("{err}"))) }) } -} \ No newline at end of file +} diff --git a/crates/typst-kit/src/package_downloads/mod.rs b/crates/typst-kit/src/package_downloads/mod.rs index 56c87591..148df394 100644 --- a/crates/typst-kit/src/package_downloads/mod.rs +++ b/crates/typst-kit/src/package_downloads/mod.rs @@ -1,33 +1,39 @@ +use ecow::{eco_format, EcoString}; use std::collections::VecDeque; use std::fmt::Debug; use std::path::{Path, PathBuf}; use std::time::Instant; -use ecow::{eco_format, EcoString}; use typst_library::diag::{PackageError, PackageResult}; use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; -use crate::package_downloads::git::GitDownloader; /// The public namespace in the default Typst registry. pub const DEFAULT_NAMESPACE: &str = "preview"; /*========BEGIN DOWNLOAD METHODS DECLARATION=========*/ #[cfg(feature = "downloads_http")] -mod http; +pub mod http; #[cfg(feature = "downloads_git")] mod git; /*========END DOWNLOAD METHODS DECLARATION===========*/ /// Trait abstraction for package a downloader. -pub trait PackageDownloader : Debug + Sync + Send { - +pub trait PackageDownloader: Debug + Sync + Send { /// Download the repository index and returns the /// list of PackageInfo elements contained in it. - fn download_index(&self, spec: &VersionlessPackageSpec) -> Result, EcoString>; + fn download_index( + &self, + spec: &VersionlessPackageSpec, + ) -> Result, EcoString>; /// Download a package from a remote repository/registry /// and writes it in the file system cache directory - fn download(&self, spec: &PackageSpec, package_dir: &Path, progress: &mut dyn Progress) -> PackageResult<()>; + fn download( + &self, + spec: &PackageSpec, + package_dir: &Path, + progress: &mut dyn Progress, + ) -> PackageResult<()>; } /// The current state of an in progress or finished download. @@ -58,13 +64,12 @@ pub trait Progress { /// The downloader object used for downloading packages #[derive(Debug)] -pub struct Downloader{ +pub struct Downloader { ///List of all available downloaders which can be instantiated at runtime http_downloader: Option>, git_downloader: Option>, } - impl Downloader { /// Construct the Downloader object instantiating all the available methods. /// The methods can be compile-time selected by features. @@ -76,31 +81,57 @@ impl Downloader { } /// Creation function for the HTTP(S) download method - fn make_http_downloader(cert: Option) -> Option>{ + fn make_http_downloader(cert: Option) -> Option> { #[cfg(not(feature = "downloads_http"))] - { None } + { + None + } #[cfg(feature = "downloads_http")] { - let user_agent = concat!("typst/", env!("CARGO_PKG_VERSION")); match cert { - Some(cert_path) => Some(Box::new(http::HttpDownloader::with_path(user_agent, cert_path))), - None => Some(Box::new(http::HttpDownloader::new(user_agent))), + Some(cert_path) => Some(Box::new(http::HttpDownloader::with_path( + http::HttpDownloader::default_user_agent(), + cert_path, + ))), + None => Some(Box::new(http::HttpDownloader::new( + http::HttpDownloader::default_user_agent(), + ))), } } } + fn get_http_downloader(&self) -> Result<&dyn PackageDownloader, PackageError> { + let reference = self.http_downloader.as_ref().ok_or_else(|| { + PackageError::Other(Some(EcoString::from( + "Http downloader has not been initialized correctly", + ))) + })?; + Ok(&**reference) + } + /// Creation function for the GIT clone method - fn make_git_downloader(_cert: Option) -> Option>{ + fn make_git_downloader(_cert: Option) -> Option> { #[cfg(not(feature = "downloads_git"))] - { None } + { + None + } #[cfg(feature = "downloads_git")] { - Some(Box::new(GitDownloader::new())) + Some(Box::new(git::GitDownloader::new())) } } + fn get_git_downloader(&self) -> Result<&dyn PackageDownloader, PackageError> { + let reference = self.git_downloader.as_ref().ok_or_else(|| { + PackageError::Other(Some(EcoString::from( + "Http downloader has not been initialized correctly", + ))) + })?; + Ok(&**reference) + } + /// Returns the correct downloader in function of the package namespace. /// The remote location of a package is encoded in its namespace in the form /// @: @@ -108,37 +139,43 @@ impl Downloader { /// It's the downloader instance's job to parse the source path in any substructure. /// /// NOTE: Treating @preview as a special case of the https downloader. - fn get_downloader(&self, ns: &str) -> Result<&Box, PackageError> { - let download_type = ns.splitn(2, ":").next(); + fn get_downloader(&self, ns: &str) -> Result<&dyn PackageDownloader, PackageError> { + let download_type = ns.split(":").next(); match download_type { #[cfg(feature = "downloads_http")] - Some("http") => self.http_downloader.as_ref().ok_or_else(|| PackageError::Other(Some(EcoString::from("Http downloader has not been initialized correctly")))), - #[cfg(feature = "downloads_http")] - Some("https") => self.http_downloader.as_ref().ok_or_else(|| PackageError::Other(Some(EcoString::from("Https downloader has not been initialized correctly")))), - #[cfg(feature = "downloads_http")] - Some("preview") => self.http_downloader.as_ref().ok_or_else(|| PackageError::Other(Some(EcoString::from("Https downloader has not been initialized correctly")))), + Some("http") | Some("https") | Some("preview") => self.get_http_downloader(), #[cfg(feature = "downloads_git")] - Some("git") => self.git_downloader.as_ref().ok_or_else(|| PackageError::Other(Some(EcoString::from("Git downloader has not been initialized correctly")))), + Some("git") => self.get_git_downloader(), - Some(dwld) => Err(PackageError::Other(Some(eco_format!("Unknown downloader type: {}", dwld)))), - None => Err(PackageError::Other(Some(EcoString::from("No downloader type specified")))), + Some(dwld) => Err(PackageError::Other(Some(eco_format!( + "Unknown downloader type: {}", + dwld + )))), + None => Err(PackageError::Other(Some(EcoString::from( + "No downloader type specified", + )))), } } } - impl PackageDownloader for Downloader { - fn download_index(&self, spec: &VersionlessPackageSpec) -> Result, EcoString> { + fn download_index( + &self, + spec: &VersionlessPackageSpec, + ) -> Result, EcoString> { let downloader = self.get_downloader(spec.namespace.as_str())?; downloader.download_index(spec) } - fn download(&self, spec: &PackageSpec, package_dir: &Path, progress: &mut dyn Progress) -> PackageResult<()> { + fn download( + &self, + spec: &PackageSpec, + package_dir: &Path, + progress: &mut dyn Progress, + ) -> PackageResult<()> { let downloader = self.get_downloader(spec.namespace.as_str())?; downloader.download(spec, package_dir, progress) } } - - diff --git a/crates/typst-syntax/src/package.rs b/crates/typst-syntax/src/package.rs index 8a0407aa..24eeff9e 100644 --- a/crates/typst-syntax/src/package.rs +++ b/crates/typst-syntax/src/package.rs @@ -264,17 +264,18 @@ impl Display for VersionlessPackageSpec { } fn is_namespace_valid(namespace: &str) -> bool { - if is_ident(namespace){ + if is_ident(namespace) { //standard namespace - return true + return true; } //if not ident, the namespace should be formed as @: let mut tokenized = namespace.splitn(2, ":"); //package type - if tokenized.next().is_none_or(|x| !is_ident(x)) { - return false + let package_remote_type = tokenized.next(); + if package_remote_type.is_none() || !is_ident(package_remote_type.unwrap()) { + return false; } //the package_path parsing is left to the downloader implementation From 236cb0884f3b447e835d9ceb65ffa2088a435aa1 Mon Sep 17 00:00:00 2001 From: Stefano Fontana Date: Sun, 15 Dec 2024 12:56:07 +0100 Subject: [PATCH 6/8] enhanced documentation --- crates/typst-kit/src/lib.rs | 9 +++---- crates/typst-kit/src/package_downloads/mod.rs | 24 ++++++++++++++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/crates/typst-kit/src/lib.rs b/crates/typst-kit/src/lib.rs index 6b6fcf10..bdcf6c93 100644 --- a/crates/typst-kit/src/lib.rs +++ b/crates/typst-kit/src/lib.rs @@ -10,13 +10,10 @@ //! - For text: Libertinus Serif, New Computer Modern //! - For math: New Computer Modern Math //! - For code: Deja Vu Sans Mono -//! - [download] contains functionality for making simple web requests with -//! status reporting, useful for downloading packages from package registries. -//! It is enabled by the `downloads` feature flag, additionally the -//! `vendor-openssl` can be used on operating systems other than macOS and -//! Windows to vendor OpenSSL when building. +//! - [package_downloads] contains functionality for handling package downloading +//! It is enabled by the `downloads` feature flag. //! - [package] contains package storage and downloading functionality based on -//! [download]. It is enabled by the `packages` feature flag and implies the +//! [package_downloads]. It is enabled by the `packages` feature flag and implies the //! `downloads` feature flag. #[cfg(feature = "fonts")] diff --git a/crates/typst-kit/src/package_downloads/mod.rs b/crates/typst-kit/src/package_downloads/mod.rs index 148df394..a82fc27f 100644 --- a/crates/typst-kit/src/package_downloads/mod.rs +++ b/crates/typst-kit/src/package_downloads/mod.rs @@ -1,3 +1,25 @@ +//! This module provides the package downloader abstraction needed +//! for remote package handling. +//! +//! # Content +//! +//! ## Traits +//! The [PackageDownloader] trait provides the abstraction needed to implement +//! multiple download method handlers. +//! Each method must allow for a package download to the local filesystem and it should provide a +//! method for downloading the repository index if it exists. +//! +//! The [Progress] trait allows for the implementation of a progress reporting struct. +//! +//! ## Module +//! [http] contains functionality for making simple web requests with status reporting, +//! useful for downloading packages from package registries. +//! It is enabled by the `downloads_http` feature flag. +//! Additionally the `vendor-openssl` can be used on operating systems other than macOS +//! and Windows to vendor OpenSSL when building. +//! +//! [git] contains functionality for handling package downloads through git repositories. + use ecow::{eco_format, EcoString}; use std::collections::VecDeque; use std::fmt::Debug; @@ -14,7 +36,7 @@ pub const DEFAULT_NAMESPACE: &str = "preview"; pub mod http; #[cfg(feature = "downloads_git")] -mod git; +pub mod git; /*========END DOWNLOAD METHODS DECLARATION===========*/ /// Trait abstraction for package a downloader. From a295495dc515c304b2217b7a5e4f411b9394cb88 Mon Sep 17 00:00:00 2001 From: Stefano Fontana Date: Sun, 15 Dec 2024 13:05:28 +0100 Subject: [PATCH 7/8] add git downloader default impl --- crates/typst-kit/src/package_downloads/git.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/typst-kit/src/package_downloads/git.rs b/crates/typst-kit/src/package_downloads/git.rs index d8005c65..c6f7debc 100644 --- a/crates/typst-kit/src/package_downloads/git.rs +++ b/crates/typst-kit/src/package_downloads/git.rs @@ -13,6 +13,12 @@ use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec}; #[derive(Debug)] pub struct GitDownloader; +impl Default for GitDownloader { + fn default() -> Self { + Self::new() + } +} + impl GitDownloader { pub fn new() -> Self { Self {} From 3962be8ebfcdf68648a4e18ee4780a7e993f6192 Mon Sep 17 00:00:00 2001 From: Stefano Fontana Date: Sun, 15 Dec 2024 18:30:39 +0100 Subject: [PATCH 8/8] migrating from git2 to gitoxide crate for git downloads --- Cargo.lock | 1182 +++++++++++++++-- Cargo.toml | 3 +- crates/typst-kit/Cargo.toml | 5 +- crates/typst-kit/src/package_downloads/git.rs | 57 +- 4 files changed, 1128 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e35b335..3a479e79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -29,6 +30,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -111,6 +118,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayref" version = "0.3.9" @@ -129,17 +142,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" -[[package]] -name = "auth-git2" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3810b5af212b013fe7302b12d86616c6c39a48e18f2e4b812a5a9e5710213791" -dependencies = [ - "dirs", - "git2", - "terminal-prompt", -] - [[package]] name = "autocfg" version = "1.4.0" @@ -222,6 +224,17 @@ dependencies = [ "wyz", ] +[[package]] +name = "bstr" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -252,6 +265,18 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" +[[package]] +name = "bytes" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" + +[[package]] +name = "bytesize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" + [[package]] name = "cc" version = "1.1.24" @@ -402,6 +427,12 @@ dependencies = [ "roff", ] +[[package]] +name = "clru" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd0f76e066e64fdc5631e3bb46381254deab9ef1158292f27c8c57e3bf3fe59" + [[package]] name = "cobs" version = "0.2.3" @@ -574,6 +605,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-url" version = "0.3.1" @@ -638,6 +683,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "ecow" version = "0.2.2" @@ -665,6 +716,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "enum-ordinalize" version = "4.3.0" @@ -727,6 +787,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd2e7510819d6fbf51a5545c8f922716ecfb14df168a3242f7d33e0239efe6a1" +[[package]] +name = "faster-hex" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183" +dependencies = [ + "serde", +] + [[package]] name = "fastrand" version = "2.1.1" @@ -875,20 +944,861 @@ dependencies = [ ] [[package]] -name = "git2" -version = "0.19.0" +name = "gix" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" +checksum = "b04c66359b5e17f92395abc433861df0edf48f39f3f590818d1d7217327dd6a1" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "once_cell", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-actor" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b24171f514cef7bb4dfb72a0b06dacf609b33ba8ad2489d4c4559a03b7afb3" +dependencies = [ + "bstr", + "gix-date", + "gix-utils", + "itoa", + "thiserror 2.0.7", + "winnow", +] + +[[package]] +name = "gix-archive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a5a8c89ece1375ca7299c80cda039d2a0a5837f2d1d576bb9259e0c6fadad5" +dependencies = [ + "bstr", + "gix-date", + "gix-object", + "gix-worktree-stream", + "jiff", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-attributes" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf9bf852194c0edfe699a2d36422d2c1f28f73b7c6d446c3f0ccd3ba232cadc" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.7", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d48b897b4bbc881aea994b4a5bbb340a04979d7be9089791304e04a9fbc66b53" +dependencies = [ + "thiserror 2.0.7", +] + +[[package]] +name = "gix-chunk" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ffbeb3a5c0b8b84c3fe4133a6f8c82fa962f4caefe8d0762eced025d3eb4f7" +dependencies = [ + "thiserror 2.0.7", +] + +[[package]] +name = "gix-command" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7d6b8f3a64453fd7e8191eb80b351eb7ac0839b40a1237cd2c137d5079fe53" +dependencies = [ + "bstr", + "gix-path", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8da6591a7868fb2b6dabddea6b09988b0b05e0213f938dbaa11a03dd7a48d85" +dependencies = [ + "bstr", + "gix-chunk", + "gix-features", + "gix-hash", + "memmap2", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-config" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6649b406ca1f99cb148959cf00468b231f07950f8ec438cc0903cda563606f19" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "memchr", + "once_cell", + "smallvec", + "thiserror 2.0.7", + "unicode-bom", + "winnow", +] + +[[package]] +name = "gix-config-value" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aaeef5d98390a3bcf9dbc6440b520b793d1bf3ed99317dc407b02be995b28e" dependencies = [ "bitflags 2.6.0", + "bstr", + "gix-path", "libc", - "libgit2-sys", - "log", - "openssl-probe", - "openssl-sys", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-credentials" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2be87bb8685fc7e6e7032ef71c45068ffff609724a0c897b8047fde10db6ae71" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-date" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "691142b1a34d18e8ed6e6114bc1a2736516c5ad60ef3aa9bd1b694886e3ca92d" +dependencies = [ + "bstr", + "itoa", + "jiff", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-diff" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a327be31a392144b60ab0b1c863362c32a1c8f7effdfa2141d5d5b6b916ef3bf" +dependencies = [ + "bstr", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "imara-diff", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-dir" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd6a0618958f9cce78a32724f8e06c4f4a57ca7080f645736d53676dc9b4db9" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-discover" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83bf6dfa4e266a4a9becb4d18fc801f92c3f7cc6c433dd86fdadbcf315ffb6ef" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-hash", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-features" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d85d673f2e022a340dba4713bed77ef2cf4cd737d2f3e0f159d45e0935fd81f" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "flate2", + "gix-hash", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "sha1_smol", + "thiserror 2.0.7", + "walkdir", +] + +[[package]] +name = "gix-filter" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5108cc58d58b27df10ac4de7f31b2eb96d588a33e5eba23739b865f5d8db7995" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline-blocking", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-fs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34740384d8d763975858fa2c176b68652a6fcc09f616e24e3ce967b0d370e4d8" +dependencies = [ + "fastrand", + "gix-features", + "gix-utils", +] + +[[package]] +name = "gix-glob" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf69a6bec0a3581567484bf99a4003afcaf6c469fd4214352517ea355cf3435" +dependencies = [ + "bitflags 2.6.0", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5eccc17194ed0e67d49285e4853307e4147e95407f91c1c3e4a13ba9f4e4ce" +dependencies = [ + "faster-hex", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-hashtable" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef65b256631078ef733bc5530c4e6b1c2e7d5c2830b75d4e9034ab3997d18fe" +dependencies = [ + "gix-hash", + "hashbrown 0.14.5", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b1fb24d2a4af0aa7438e2771d60c14a80cf2c9bd55c29cf1712b841f05bb8a" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-index" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "270645fd20556b64c8ffa1540d921b281e6994413a0ca068596f97e9367a257a" +dependencies = [ + "bitflags 2.6.0", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.14.5", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-lock" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd3ab68a452db63d9f3ebdacb10f30dba1fa0d31ac64f4203d395ed1102d940" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-mailmap" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6a108b866e00b8a59b8746906cccf2648ffc3e393dc9cca97254dd75c2ddf8c" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-negotiate" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27f830a16405386e9c83b9d5be8261fe32bbd6b3caf15bd1b284c6b2b7ef1a8" +dependencies = [ + "bitflags 2.6.0", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-object" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65d93e2bbfa83a307e47f45e45de7b6c04d7375a8bd5907b215f4bf45237d879" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.7", + "winnow", +] + +[[package]] +name = "gix-odb" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93bed6e1b577c25a6bb8e6ecbf4df525f29a671ddf5f2221821a56a8dbeec4e3" +dependencies = [ + "arc-swap", + "gix-date", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "parking_lot", + "tempfile", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-pack" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b91fec04d359544fecbb8e85117ec746fbaa9046ebafcefb58cb74f20dc76d4" +dependencies = [ + "clru", + "gix-chunk", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "memmap2", + "parking_lot", + "smallvec", + "thiserror 2.0.7", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a720e5bebf494c3ceffa85aa89f57a5859450a0da0a29ebe89171e23543fa78" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-packetline-blocking" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce9004ce1bc00fd538b11c1ec8141a1558fb3af3d2b7ac1ac5c41881f9e42d2a" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-path" +version = "0.10.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc292ef1a51e340aeb0e720800338c805975724c1dfbd243185452efd8645b7" +dependencies = [ + "bstr", + "gix-trace", + "home", + "once_cell", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-pathspec" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c472dfbe4a4e96fcf7efddcd4771c9037bb4fdea2faaabf2f4888210c75b81e" +dependencies = [ + "bitflags 2.6.0", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-prompt" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7822afc4bc9c5fbbc6ce80b00f41c129306b7685cac3248dbfa14784960594" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-protocol" +version = "0.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7e7e51a0dea531d3448c297e2fa919b2de187111a210c324b7e9f81508b8ca" +dependencies = [ + "bstr", + "gix-credentials", + "gix-date", + "gix-features", + "gix-hash", + "gix-transport", + "gix-utils", + "maybe-async", + "thiserror 2.0.7", + "winnow", +] + +[[package]] +name = "gix-quote" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a1e282216ec2ab2816cd57e6ed88f8009e634aec47562883c05ac8a7009a63" +dependencies = [ + "bstr", + "gix-utils", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-ref" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eae462723686272a58f49501015ef7c0d67c3e042c20049d8dd9c7eff92efde" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.7", + "winnow", +] + +[[package]] +name = "gix-refspec" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00c056bb747868c7eb0aeb352c9f9181ab8ca3d0a2550f16470803500c6c413d" +dependencies = [ + "bstr", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-revision" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44488e0380847967bc3e3cacd8b22652e02ea1eb58afb60edd91847695cd2d8d" +dependencies = [ + "bitflags 2.6.0", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-revwalk" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "510026fc32f456f8f067d8f37c34088b97a36b2229d88a6a5023ef179fcb109d" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-sec" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8b876ef997a955397809a2ec398d6a45b7a55b4918f2446344330f778d14fd6" +dependencies = [ + "bitflags 2.6.0", + "gix-path", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "gix-status" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201396192ee4c4dd9e8a84fed4b0d2b33d639fca815fb99b0f653dfeddf38585" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-submodule" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2455f8c0fcb6ebe2a6e83c8f522d30615d763eb2ef7a23c7d929f9476e89f5c" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-tempfile" +version = "15.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2feb86ef094cc77a4a9a5afbfe5de626897351bbbd0de3cb9314baf3049adb82" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "once_cell", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04bdde120c29f1fc23a24d3e115aeeea3d60d8e65bab92cc5f9d90d9302eb952" + +[[package]] +name = "gix-transport" +version = "0.43.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39a1a41357b7236c03e0c984147f823d87c3e445a8581bac7006df141577200b" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-traverse" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff2ec9f779680f795363db1c563168b32b8d6728ec58564c628e85c92d29faf" +dependencies = [ + "bitflags 2.6.0", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-url" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e09f97db3618fb8e473d7d97e77296b50aaee0ddcd6a867f07443e3e87391099" +dependencies = [ + "bstr", + "gix-features", + "gix-path", + "thiserror 2.0.7", "url", ] +[[package]] +name = "gix-utils" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba427e3e9599508ed98a6ddf8ed05493db114564e338e41f6a996d2e4790335f" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd520d09f9f585b34b32aba1d0b36ada89ab7fefb54a8ca3fe37fc482a750937" +dependencies = [ + "bstr", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-worktree" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756dbbe15188fa22540d5eab941f8f9cf511a5364d5aec34c88083c09f4bea13" +dependencies = [ + "bstr", + "gix-attributes", + "gix-features", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebd5eead61d37b334bc31810c9980aa72d659044513cae0e342a88fed2c22ba" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.7", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1879375cbd896823237480a3034aa9fa056b4f9dc9f35a4ac8cd4dc052b2793" +dependencies = [ + "gix-attributes", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", + "thiserror 2.0.7", +] + [[package]] name = "half" version = "2.4.1" @@ -912,6 +1822,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", + "allocator-api2", ] [[package]] @@ -934,7 +1845,7 @@ dependencies = [ "paste", "serde", "serde_yaml 0.9.34+deprecated", - "thiserror", + "thiserror 1.0.64", "unic-langid", "unicode-segmentation", "unscanny", @@ -947,12 +1858,27 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "httpdate" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "human_format" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3b1f728c459d27b12448862017b96ad4767b1ec2ec5e6434e99f1577f085b8" + [[package]] name = "hypher" version = "0.1.5" @@ -1178,6 +2104,16 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edcd27d72f2f071c64249075f42e205ff93c9a4c5f6c6da53e79ed9f9832c285" +[[package]] +name = "imara-diff" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc9da1a252bd44cd341657203722352efc9bc0c847d06ea6d2dc1cd1135e0a01" +dependencies = [ + "ahash", + "hashbrown 0.14.5", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1226,6 +2162,16 @@ dependencies = [ "libc", ] +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -1257,6 +2203,31 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jiff" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db69f08d4fb10524cacdb074c10b296299d71274ddbc830a8ee65666867002e9" +dependencies = [ + "jiff-tzdb-platform", + "windows-sys 0.59.0", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91335e575850c5c4c673b9bd467b0e025f164ca59d0564f69d0c2ee0ffad4653" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9835f0060a626fe59f160437bc725491a6af23133ea906500027d1bd2f8f4329" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jobserver" version = "0.1.32" @@ -1304,6 +2275,15 @@ dependencies = [ "libc", ] +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + [[package]] name = "kurbo" version = "0.11.1" @@ -1348,20 +2328,6 @@ dependencies = [ "cc", ] -[[package]] -name = "libgit2-sys" -version = "0.17.0+1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" -dependencies = [ - "cc", - "libc", - "libssh2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", -] - [[package]] name = "libm" version = "0.2.8" @@ -1379,32 +2345,6 @@ dependencies = [ "redox_syscall", ] -[[package]] -name = "libssh2-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1469,6 +2409,17 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "maybe-async" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "memchr" version = "2.7.4" @@ -1912,13 +2863,25 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] +[[package]] +name = "prodash" +version = "29.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a266d8d6020c61a437be704c5e618037588e1985c7dbb7bf8d265db84cffe325" +dependencies = [ + "bytesize", + "human_format", + "log", + "parking_lot", +] + [[package]] name = "psm" version = "0.1.23" @@ -2048,7 +3011,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.64", ] [[package]] @@ -2316,18 +3279,49 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "shell-escape" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "sigpipe" version = "0.1.3" @@ -2404,6 +3398,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strict-num" version = "0.1.1" @@ -2490,9 +3490,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -2527,7 +3527,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "thiserror", + "thiserror 1.0.64", "walkdir", "yaml-rust", ] @@ -2571,16 +3571,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "terminal-prompt" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572818b3472910acbd5dff46a3413715c18e934b071ab2ba464a7b2c2af16376" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "terminal_size" version = "0.4.0" @@ -2603,7 +3593,16 @@ version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.64", +] + +[[package]] +name = "thiserror" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" +dependencies = [ + "thiserror-impl 2.0.7", ] [[package]] @@ -2617,6 +3616,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.3.36" @@ -2930,13 +3940,12 @@ dependencies = [ name = "typst-kit" version = "0.12.0" dependencies = [ - "auth-git2", "dirs", "ecow", "env_proxy", "flate2", "fontdb", - "git2", + "gix", "native-tls", "once_cell", "openssl", @@ -3182,6 +4191,15 @@ dependencies = [ "thin-vec", ] +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + [[package]] name = "unic-langid" version = "0.9.5" @@ -3222,6 +4240,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64af057ad7466495ca113126be61838d8af947f41d93a949980b2389a118082f" +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + [[package]] name = "unicode-ccc" version = "0.3.0" @@ -3883,7 +4907,7 @@ dependencies = [ "flate2", "indexmap 2.6.0", "memchr", - "thiserror", + "thiserror 1.0.64", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index f6605b0d..556681f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,6 @@ typst-timing = { path = "crates/typst-timing", version = "0.12.0" } typst-utils = { path = "crates/typst-utils", version = "0.12.0" } typst-assets = { git = "https://github.com/typst/typst-assets", rev = "8cccef9" } typst-dev-assets = { git = "https://github.com/typst/typst-dev-assets", rev = "b07d156" } -auth-git2 = "0.5.5" arrayvec = "0.7.4" az = "1.2" base64 = "0.22" @@ -59,7 +58,7 @@ env_proxy = "0.4" flate2 = "1" fontdb = { version = "0.21", default-features = false } fs_extra = "1.3" -git2 = "0.19.0" +gix = "0.68.0" hayagriva = "0.8" heck = "0.5" hypher = "0.1.4" diff --git a/crates/typst-kit/Cargo.toml b/crates/typst-kit/Cargo.toml index 94f3fa40..a751d97d 100644 --- a/crates/typst-kit/Cargo.toml +++ b/crates/typst-kit/Cargo.toml @@ -25,8 +25,7 @@ native-tls = { workspace = true, optional = true } once_cell = { workspace = true } tar = { workspace = true, optional = true } ureq = { workspace = true, optional = true } -git2 = { workspace = true, optional = true } -auth-git2 = { workspace = true, optional = true } +gix = { workspace = true, optional = true, features = ["worktree-mutation", "blocking-network-client"] } # Explicitly depend on OpenSSL if applicable, so that we can add the # `openssl/vendored` feature to it if `vendor-openssl` is enabled. @@ -42,7 +41,7 @@ fonts = ["dep:fontdb", "fontdb/memmap", "fontdb/fontconfig"] # Add generic downloading utilities downloads = ["downloads_http", "downloads_git"] downloads_http = ["dep:env_proxy", "dep:native-tls", "dep:ureq", "dep:openssl"] -downloads_git = ["git2", "auth-git2"] +downloads_git = ["gix"] # Add package downloading utilities, implies `downloads` packages = ["downloads", "dep:dirs", "dep:flate2", "dep:tar"] diff --git a/crates/typst-kit/src/package_downloads/git.rs b/crates/typst-kit/src/package_downloads/git.rs index c6f7debc..721311ad 100644 --- a/crates/typst-kit/src/package_downloads/git.rs +++ b/crates/typst-kit/src/package_downloads/git.rs @@ -1,10 +1,8 @@ use crate::package_downloads::{DownloadState, PackageDownloader, Progress}; -use auth_git2::GitAuthenticator; use ecow::{eco_format, EcoString}; -use git2::build::RepoBuilder; -use git2::{FetchOptions, RemoteCallbacks}; -use std::collections::VecDeque; +use gix::remote::fetch::Shallow; use std::fmt::Debug; +use std::num::NonZero; use std::path::Path; use std::time::Instant; use typst_library::diag::{PackageError, PackageResult}; @@ -32,45 +30,34 @@ impl GitDownloader { progress: &mut dyn Progress, ) -> Result<(), EcoString> { progress.print_start(); - - eprintln!("{} {} {}", repo, tag, dest.display()); - let state = DownloadState { content_len: None, total_downloaded: 0, - bytes_per_second: VecDeque::from(vec![0; 5]), + bytes_per_second: Default::default(), start_time: Instant::now(), }; - let auth = GitAuthenticator::default(); - let git_config = git2::Config::open_default() - .map_err(|err| EcoString::from(format!("{err}")))?; + std::fs::create_dir_all(dest).map_err(|x| eco_format!("{x}"))?; + let url = gix::url::parse(repo.into()).map_err(|x| eco_format!("{x}"))?; + let mut prepare_fetch = + gix::prepare_clone(url, dest).map_err(|x| eco_format!("{x}"))?; + prepare_fetch = prepare_fetch + .with_shallow(Shallow::DepthAtRemote(NonZero::new(1).unwrap())) + .with_ref_name(Some(tag)) + .map_err(|x| eco_format!("{x}"))?; - let mut fetch_options = FetchOptions::new(); - let mut remote_callbacks = RemoteCallbacks::new(); - - remote_callbacks.credentials(auth.credentials(&git_config)); - fetch_options.remote_callbacks(remote_callbacks); - - let repo = RepoBuilder::new() - .fetch_options(fetch_options) - .clone(repo, dest) - .map_err(|err| EcoString::from(format!("{err}")))?; - - let (object, reference) = repo - .revparse_ext(tag) - .map_err(|err| EcoString::from(format!("{err}")))?; - repo.checkout_tree(&object, None) - .map_err(|err| EcoString::from(format!("{err}")))?; - - match reference { - // gref is an actual reference like branches or tags - Some(gref) => repo.set_head(gref.name().unwrap()), - // this is a commit, not a reference - None => repo.set_head_detached(object.id()), + let (mut prepare_checkout, _) = prepare_fetch + .fetch_then_checkout(gix::progress::Discard, &gix::interrupt::IS_INTERRUPTED) + .map_err(|x| eco_format!("{x}"))?; + if prepare_checkout.repo().work_dir().is_none() { + return Err(eco_format!( + "Cloned git repository but files are not available." + ))?; } - .map_err(|err| EcoString::from(format!("{err}")))?; + prepare_checkout + .main_worktree(gix::progress::Discard, &gix::interrupt::IS_INTERRUPTED) + .map_err(|x| eco_format!("{x}"))?; progress.print_finish(&state); Ok(()) } @@ -122,7 +109,7 @@ impl PackageDownloader for GitDownloader { ) -> PackageResult<()> { let repo = Self::parse_namespace(spec.namespace.as_str(), spec.name.as_str()) .map_err(|x| PackageError::Other(Some(x)))?; - let tag = format!("v{}", spec.version); + let tag = format!("refs/tags/v{}", spec.version); self.download_with_progress(repo.as_str(), tag.as_str(), package_dir, progress) .map_err(|x| PackageError::Other(Some(x))) }