merge git packages

Cargo.toml

@@ -58,6 +58,7 @@ env_proxy = "0.4"
 flate2 = "1"
 fontdb = { version = "0.21", default-features = false }
 fs_extra = "1.3"
+gix = "0.68.0"
 hayagriva = "0.8"
 heck = "0.5"
 hypher = "0.1.4"

crates/typst-cli/Cargo.toml

@@ -21,7 +21,7 @@ doc = false
 typst = { workspace = true }
 typst-eval = { workspace = true }
 typst-html = { workspace = true }
-typst-kit = { workspace = true }
+typst-kit = { workspace = true, features = ["downloads_http"] }
 typst-macros = { workspace = true }
 typst-pdf = { workspace = true }
 typst-render = { workspace = true }

crates/typst-cli/src/download.rs

@@ -6,7 +6,7 @@ use std::time::{Duration, Instant};
 use codespan_reporting::term;
 use codespan_reporting::term::termcolor::WriteColor;
 use typst::utils::format_duration;
-use typst_kit::download::{DownloadState, Downloader, Progress};
+use typst_kit::package_downloads::{DownloadState, Downloader, Progress};
 
 use crate::terminal::{self, TermOut};
 use crate::ARGS;
@@ -43,11 +43,7 @@ impl<T: Display> Progress for PrintDownload<T> {
 
 /// Returns a new downloader.
 pub fn downloader() -> Downloader {
-    let user_agent = concat!("typst/", env!("CARGO_PKG_VERSION"));
+    Downloader::new(ARGS.cert.clone())
-    match ARGS.cert.clone() {
-        Some(cert) => Downloader::with_path(user_agent, cert),
-        None => Downloader::new(user_agent),
-    }
 }
 
 /// Compile and format several download statistics and make and attempt at

crates/typst-cli/src/update.rs

@@ -7,12 +7,12 @@ use semver::Version;
 use serde::Deserialize;
 use tempfile::NamedTempFile;
 use typst::diag::{bail, StrResult};
-use typst_kit::download::Downloader;
+use typst_kit::package_downloads::http::HttpDownloader;
 use xz2::bufread::XzDecoder;
 use zip::ZipArchive;
 
 use crate::args::UpdateCommand;
-use crate::download::{self, PrintDownload};
+use crate::download::PrintDownload;
 
 const TYPST_GITHUB_ORG: &str = "typst";
 const TYPST_REPO: &str = "typst";
@@ -91,7 +91,8 @@ pub fn update(command: &UpdateCommand) -> StrResult<()> {
     fs::copy(current_exe, &backup_path)
         .map_err(|err| eco_format!("failed to create backup ({err})"))?;
 
-    let downloader = download::downloader();
+    //no certificate is needed to download from GitHub
+    let downloader = HttpDownloader::new(HttpDownloader::default_user_agent());
 
     let release = Release::from_tag(command.version.as_ref(), &downloader)?;
     if !update_needed(&release)? && !command.force {
@@ -133,7 +134,7 @@ impl Release {
     /// Typst repository.
     pub fn from_tag(
         tag: Option<&Version>,
-        downloader: &Downloader,
+        downloader: &HttpDownloader,
     ) -> StrResult<Release> {
         let url = match tag {
             Some(tag) => format!(
@@ -144,7 +145,7 @@ impl Release {
             ),
         };
 
-        match downloader.download(&url) {
+        match downloader.perform_download(&url) {
             Ok(response) => response.into_json().map_err(|err| {
                 eco_format!("failed to parse release information ({err})")
             }),
@@ -161,7 +162,7 @@ impl Release {
     pub fn download_binary(
         &self,
         asset_name: &str,
-        downloader: &Downloader,
+        downloader: &HttpDownloader,
     ) -> StrResult<Vec<u8>> {
         let asset = self.assets.iter().find(|a| a.name.starts_with(asset_name)).ok_or(
             eco_format!(

crates/typst-kit/Cargo.toml

@@ -25,6 +25,7 @@ native-tls = { workspace = true, optional = true }
 once_cell = { workspace = true }
 tar = { workspace = true, optional = true }
 ureq = { workspace = true, optional = true }
+gix = { workspace = true, optional = true, features = ["worktree-mutation", "blocking-network-client"] }
 
 # Explicitly depend on OpenSSL if applicable, so that we can add the
 # `openssl/vendored` feature to it if `vendor-openssl` is enabled.
@@ -38,7 +39,9 @@ default = ["fonts", "packages"]
 fonts = ["dep:fontdb", "fontdb/memmap", "fontdb/fontconfig"]
 
 # Add generic downloading utilities
-downloads = ["dep:env_proxy", "dep:native-tls", "dep:ureq", "dep:openssl"]
+downloads = ["downloads_http", "downloads_git"]
+downloads_http = ["dep:env_proxy", "dep:native-tls", "dep:ureq", "dep:openssl"]
+downloads_git = ["gix"]
 
 # Add package downloading utilities, implies `downloads`
 packages = ["downloads", "dep:dirs", "dep:flate2", "dep:tar"]

crates/typst-kit/src/lib.rs

@@ -10,18 +10,15 @@
 //!   - For text: Libertinus Serif, New Computer Modern
 //!   - For math: New Computer Modern Math
 //!   - For code: Deja Vu Sans Mono
-//! - [download] contains functionality for making simple web requests with
+//! - [package_downloads] contains functionality for handling package downloading
-//!   status reporting, useful for downloading packages from package registries.
+//!   It is enabled by the `downloads` feature flag.
-//!   It is enabled by the `downloads` feature flag, additionally the
-//!   `vendor-openssl` can be used on operating systems other than macOS and
-//!   Windows to vendor OpenSSL when building.
 //! - [package] contains package storage and downloading functionality based on
-//!   [download]. It is enabled by the `packages` feature flag and implies the
+//!   [package_downloads]. It is enabled by the `packages` feature flag and implies the
 //!   `downloads` feature flag.
 
-#[cfg(feature = "downloads")]
-pub mod download;
 #[cfg(feature = "fonts")]
 pub mod fonts;
 #[cfg(feature = "packages")]
 pub mod package;
+#[cfg(feature = "downloads")]
+pub mod package_downloads;

crates/typst-kit/src/package.rs

@@ -1,23 +1,14 @@
 //! Download and unpack packages and package indices.
-
-use std::fs;
 use std::path::{Path, PathBuf};
 
+use crate::package_downloads::{Downloader, PackageDownloader, Progress};
 use ecow::eco_format;
 use once_cell::sync::OnceCell;
-use typst_library::diag::{bail, PackageError, PackageResult, StrResult};
+use typst_library::diag::{PackageError, PackageResult, StrResult};
 use typst_syntax::package::{
     PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec,
 };
 
-use crate::download::{Downloader, Progress};
-
-/// The default Typst registry.
-pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org";
-
-/// The public namespace in the default Typst registry.
-pub const DEFAULT_NAMESPACE: &str = "preview";
-
 /// The default packages sub directory within the package and package cache paths.
 pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
 
@@ -93,25 +84,27 @@ impl PackageStorage {
             }
         }
 
+        // check the package_path for the package directory.
         if let Some(packages_dir) = &self.package_path {
             let dir = packages_dir.join(&subdir);
             if dir.exists() {
+                // no need to download, already in the path.
                 return Ok(dir);
             }
         }
 
+        // package was not in the package_path. check if it has been cached
         if let Some(cache_dir) = &self.package_cache_path {
             let dir = cache_dir.join(&subdir);
             if dir.exists() {
+                //package was cached, so return the cached directory
                 return Ok(dir);
             }
 
             // Download from network if it doesn't exist yet.
-            if spec.namespace == DEFAULT_NAMESPACE {
+            self.download_package(spec, &dir, progress)?;
-                self.download_package(spec, &dir, progress)?;
+            if dir.exists() {
-                if dir.exists() {
+                return Ok(dir);
-                    return Ok(dir);
-                }
             }
         }
 
@@ -123,47 +116,39 @@ impl PackageStorage {
         &self,
         spec: &VersionlessPackageSpec,
     ) -> StrResult<PackageVersion> {
-        if spec.namespace == DEFAULT_NAMESPACE {
+        // Same logical flow as per package download. Check package path, then check online.
-            // For `DEFAULT_NAMESPACE`, download the package index and find the latest
+        // Do not check in the data directory because the latter is not intended for storage
-            // version.
+        // of local packages.
-            self.download_index()?
+        let subdir = format!("{}/{}", spec.namespace, spec.name);
-                .iter()
+        let res = self
-                .filter(|package| package.name == spec.name)
+            .package_path
-                .map(|package| package.version)
+            .iter()
-                .max()
+            .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
-                .ok_or_else(|| eco_format!("failed to find package {spec}"))
+            .flatten()
-        } else {
+            .filter_map(|entry| entry.ok())
-            // For other namespaces, search locally. We only search in the data
+            .map(|entry| entry.path())
-            // directory and not the cache directory, because the latter is not
+            .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
-            // intended for storage of local packages.
+            .max();
-            let subdir = format!("{}/{}", spec.namespace, spec.name);
+
-            self.package_path
+        if let Some(version) = res {
-                .iter()
+            return Ok(version);
-                .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
-                .flatten()
-                .filter_map(|entry| entry.ok())
-                .map(|entry| entry.path())
-                .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
-                .max()
-                .ok_or_else(|| eco_format!("please specify the desired version"))
         }
+
+        self.download_index(spec)?
+            .iter()
+            .filter(|package| package.name == spec.name)
+            .map(|package| package.version)
+            .max()
+            .ok_or_else(|| eco_format!("failed to find package {spec}"))
     }
 
     /// Download the package index. The result of this is cached for efficiency.
-    pub fn download_index(&self) -> StrResult<&[PackageInfo]> {
+    pub fn download_index(
+        &self,
+        spec: &VersionlessPackageSpec,
+    ) -> StrResult<&[PackageInfo]> {
         self.index
-            .get_or_try_init(|| {
+            .get_or_try_init(|| self.downloader.download_index(spec))
-                let url = format!("{DEFAULT_REGISTRY}/{DEFAULT_NAMESPACE}/index.json");
-                match self.downloader.download(&url) {
-                    Ok(response) => response.into_json().map_err(|err| {
-                        eco_format!("failed to parse package index: {err}")
-                    }),
-                    Err(ureq::Error::Status(404, _)) => {
-                        bail!("failed to fetch package index (not found)")
-                    }
-                    Err(err) => bail!("failed to fetch package index ({err})"),
-                }
-            })
             .map(AsRef::as_ref)
     }
 
@@ -177,31 +162,15 @@ impl PackageStorage {
         package_dir: &Path,
         progress: &mut dyn Progress,
     ) -> PackageResult<()> {
-        assert_eq!(spec.namespace, DEFAULT_NAMESPACE);
+        match self.downloader.download(spec, package_dir, progress) {
-
+            Err(PackageError::NotFound(spec)) => {
-        let url = format!(
-            "{DEFAULT_REGISTRY}/{DEFAULT_NAMESPACE}/{}-{}.tar.gz",
-            spec.name, spec.version
-        );
-
-        let data = match self.downloader.download_with_progress(&url, progress) {
-            Ok(data) => data,
-            Err(ureq::Error::Status(404, _)) => {
                 if let Ok(version) = self.determine_latest_version(&spec.versionless()) {
-                    return Err(PackageError::VersionNotFound(spec.clone(), version));
+                    Err(PackageError::VersionNotFound(spec.clone(), version))
                 } else {
-                    return Err(PackageError::NotFound(spec.clone()));
+                    Err(PackageError::NotFound(spec.clone()))
                 }
             }
-            Err(err) => {
+            val => val,
-                return Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))
+        }
-            }
-        };
-
-        let decompressed = flate2::read::GzDecoder::new(data.as_slice());
-        tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| {
-            fs::remove_dir_all(package_dir).ok();
-            PackageError::MalformedArchive(Some(eco_format!("{err}")))
-        })
     }
 }

crates/typst-kit/src/package_downloads/git.rs

@@ -0,0 +1,116 @@
+use crate::package_downloads::{DownloadState, PackageDownloader, Progress};
+use ecow::{eco_format, EcoString};
+use gix::remote::fetch::Shallow;
+use std::fmt::Debug;
+use std::num::NonZero;
+use std::path::Path;
+use std::time::Instant;
+use typst_library::diag::{PackageError, PackageResult};
+use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec};
+
+#[derive(Debug)]
+pub struct GitDownloader;
+
+impl Default for GitDownloader {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl GitDownloader {
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    pub fn download_with_progress(
+        &self,
+        repo: &str,
+        tag: &str,
+        dest: &Path,
+        progress: &mut dyn Progress,
+    ) -> Result<(), EcoString> {
+        progress.print_start();
+        let state = DownloadState {
+            content_len: None,
+            total_downloaded: 0,
+            bytes_per_second: Default::default(),
+            start_time: Instant::now(),
+        };
+
+        std::fs::create_dir_all(dest).map_err(|x| eco_format!("{x}"))?;
+        let url = gix::url::parse(repo.into()).map_err(|x| eco_format!("{x}"))?;
+        let mut prepare_fetch =
+            gix::prepare_clone(url, dest).map_err(|x| eco_format!("{x}"))?;
+        prepare_fetch = prepare_fetch
+            .with_shallow(Shallow::DepthAtRemote(NonZero::new(1).unwrap()))
+            .with_ref_name(Some(tag))
+            .map_err(|x| eco_format!("{x}"))?;
+
+        let (mut prepare_checkout, _) = prepare_fetch
+            .fetch_then_checkout(gix::progress::Discard, &gix::interrupt::IS_INTERRUPTED)
+            .map_err(|x| eco_format!("{x}"))?;
+        if prepare_checkout.repo().work_dir().is_none() {
+            return Err(eco_format!(
+                "Cloned git repository but files are not available."
+            ))?;
+        }
+
+        prepare_checkout
+            .main_worktree(gix::progress::Discard, &gix::interrupt::IS_INTERRUPTED)
+            .map_err(|x| eco_format!("{x}"))?;
+        progress.print_finish(&state);
+        Ok(())
+    }
+
+    /// Parses the namespace of the package into the correct registry and namespace.
+    /// The namespace format is the following:
+    ///
+    /// @git:<git host and user>
+    ///
+    /// The final repository cloned will be formed by the git host and the repository name
+    /// with the adequate extension, checking out to the tag specified by the version in the format
+    /// v<major>.<minor>.<patch>
+    ///
+    /// For example, the package
+    /// @git:git@github.com:typst/package:0.0
+    /// will result in the cloning of the repository git@github.com:typst/package.git
+    /// and the checkout and detached head state at tag v0.1.0
+    ///
+    /// NOTE: no index download is possible.
+    fn parse_namespace(ns: &str, name: &str) -> Result<String, EcoString> {
+        let mut parts = ns.splitn(2, ":");
+        let schema =
+            parts.next().ok_or_else(|| eco_format!("expected schema in {}", ns))?;
+        let repo = parts
+            .next()
+            .ok_or_else(|| eco_format!("invalid package repo {}", ns))?;
+
+        if !schema.eq("git") {
+            Err(eco_format!("invalid schema in {}", ns))?
+        }
+
+        Ok(format!("{repo}/{name}.git"))
+    }
+}
+
+impl PackageDownloader for GitDownloader {
+    fn download_index(
+        &self,
+        _spec: &VersionlessPackageSpec,
+    ) -> Result<Vec<PackageInfo>, EcoString> {
+        Err(eco_format!("Downloading index is not supported for git repositories"))
+    }
+
+    fn download(
+        &self,
+        spec: &PackageSpec,
+        package_dir: &Path,
+        progress: &mut dyn Progress,
+    ) -> PackageResult<()> {
+        let repo = Self::parse_namespace(spec.namespace.as_str(), spec.name.as_str())
+            .map_err(|x| PackageError::Other(Some(x)))?;
+        let tag = format!("refs/tags/v{}", spec.version);
+        self.download_with_progress(repo.as_str(), tag.as_str(), package_dir, progress)
+            .map_err(|x| PackageError::Other(Some(x)))
+    }
+}

crates/typst-kit/src/package_downloads/http.rs

@@ -7,27 +7,24 @@
 
 use std::collections::VecDeque;
 use std::fmt::Debug;
+use std::fs;
 use std::io::{self, ErrorKind, Read};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-use ecow::EcoString;
+use crate::package_downloads::{
+    DownloadState, PackageDownloader, Progress, DEFAULT_NAMESPACE,
+};
+use ecow::{eco_format, EcoString};
 use native_tls::{Certificate, TlsConnector};
 use once_cell::sync::OnceCell;
+use typst_library::diag::{bail, PackageError, PackageResult};
+use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec};
 use ureq::Response;
 
-/// Manages progress reporting for downloads.
+/// The default Typst registry.
-pub trait Progress {
+pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org";
-    /// Invoked when a download is started.
-    fn print_start(&mut self);
-
-    /// Invoked repeatedly while a download is ongoing.
-    fn print_progress(&mut self, state: &DownloadState);
-
-    /// Invoked when a download is finished.
-    fn print_finish(&mut self, state: &DownloadState);
-}
 
 /// An implementation of [`Progress`] with no-op reporting, i.e., reporting
 /// events are swallowed.
@@ -39,28 +36,18 @@ impl Progress for ProgressSink {
     fn print_finish(&mut self, _: &DownloadState) {}
 }
 
-/// The current state of an in progress or finished download.
-#[derive(Debug)]
-pub struct DownloadState {
-    /// The expected amount of bytes to download, `None` if the response header
-    /// was not set.
-    pub content_len: Option<usize>,
-    /// The total amount of downloaded bytes until now.
-    pub total_downloaded: usize,
-    /// A backlog of the amount of downloaded bytes each second.
-    pub bytes_per_second: VecDeque<usize>,
-    /// The download starting instant.
-    pub start_time: Instant,
-}
-
 /// A minimal https client for downloading various resources.
-pub struct Downloader {
+pub struct HttpDownloader {
     user_agent: EcoString,
     cert_path: Option<PathBuf>,
     cert: OnceCell<Certificate>,
 }
 
-impl Downloader {
+impl HttpDownloader {
+    pub fn default_user_agent() -> String {
+        format!("typst-kit/{}", env!("CARGO_PKG_VERSION"))
+    }
+
     /// Crates a new downloader with the given user agent and no certificate.
     pub fn new(user_agent: impl Into<EcoString>) -> Self {
         Self {
@@ -81,15 +68,6 @@ impl Downloader {
         }
     }
 
-    /// Crates a new downloader with the given user agent and certificate.
-    pub fn with_cert(user_agent: impl Into<EcoString>, cert: Certificate) -> Self {
-        Self {
-            user_agent: user_agent.into(),
-            cert_path: None,
-            cert: OnceCell::with_value(cert),
-        }
-    }
-
     /// Returns the certificate this client is using, if a custom certificate
     /// is used it is loaded on first access.
     ///
@@ -107,7 +85,7 @@ impl Downloader {
 
     /// Download binary data from the given url.
     #[allow(clippy::result_large_err)]
-    pub fn download(&self, url: &str) -> Result<ureq::Response, ureq::Error> {
+    pub fn perform_download(&self, url: &str) -> Result<ureq::Response, ureq::Error> {
         let mut builder = ureq::AgentBuilder::new();
         let mut tls = TlsConnector::builder();
 
@@ -143,12 +121,47 @@ impl Downloader {
         progress: &mut dyn Progress,
     ) -> Result<Vec<u8>, ureq::Error> {
         progress.print_start();
-        let response = self.download(url)?;
+        let response = self.perform_download(url)?;
         Ok(RemoteReader::from_response(response, progress).download()?)
     }
+
+    /// Parses the namespace of the package into the correct registry and namespace.
+    /// The namespace format is the following:
+    ///
+    /// @http[s]:<registry host>:<namespace>/package-name>:package-version
+    ///
+    /// resulting in the package location to be resolved as
+    /// http[s]://<registry host>/<namespace>/<package-name>-<package-version>.tar.gz
+    ///
+    /// and the index to be resolved as
+    /// http[s]://<registry host>/<namespace>/index.json
+    ///
+    /// NOTE: preview namespace is treated as the namespace formed as
+    /// @https:packages.typst.org:preview/package-name>:package-version
+    fn parse_namespace(ns: &str) -> Result<(String, String), EcoString> {
+        if ns.eq(DEFAULT_NAMESPACE) {
+            return Ok((DEFAULT_REGISTRY.to_string(), DEFAULT_NAMESPACE.to_string()));
+        }
+        let mut parts = ns.splitn(3, ":");
+
+        let schema =
+            parts.next().ok_or_else(|| eco_format!("expected schema in {}", ns))?;
+        let registry = parts
+            .next()
+            .ok_or_else(|| eco_format!("invalid package registry in namespace {}", ns))?;
+        let ns = parts
+            .next()
+            .ok_or_else(|| eco_format!("invalid package namespace in {}", ns))?;
+
+        if !schema.eq("http") && !schema.eq("https") {
+            Err(eco_format!("invalid schema in {}", ns))?
+        }
+
+        Ok((format!("{schema}://{registry}"), ns.to_string()))
+    }
 }
 
-impl Debug for Downloader {
+impl Debug for HttpDownloader {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Downloader")
             .field("user_agent", &self.user_agent)
@@ -257,3 +270,48 @@ impl<'p> RemoteReader<'p> {
         Ok(data)
     }
 }
+
+impl PackageDownloader for HttpDownloader {
+    fn download_index(
+        &self,
+        spec: &VersionlessPackageSpec,
+    ) -> Result<Vec<PackageInfo>, EcoString> {
+        let (registry, namespace) = Self::parse_namespace(spec.namespace.as_str())?;
+        let url = format!("{registry}/{namespace}/index.json");
+        match self.perform_download(&url) {
+            Ok(response) => response
+                .into_json()
+                .map_err(|err| eco_format!("failed to parse package index: {err}")),
+            Err(ureq::Error::Status(404, _)) => {
+                bail!("failed to fetch package index (not found)")
+            }
+            Err(err) => bail!("failed to fetch package index ({err})"),
+        }
+    }
+
+    fn download(
+        &self,
+        spec: &PackageSpec,
+        package_dir: &Path,
+        progress: &mut dyn Progress,
+    ) -> PackageResult<()> {
+        let (registry, namespace) = Self::parse_namespace(spec.namespace.as_str())
+            .map_err(|x| PackageError::Other(Some(x)))?;
+
+        let url =
+            format!("{}/{}/{}-{}.tar.gz", registry, namespace, spec.name, spec.version);
+        let data = match self.download_with_progress(&url, progress) {
+            Ok(data) => data,
+            Err(ureq::Error::Status(404, _)) => {
+                Err(PackageError::NotFound(spec.clone()))?
+            }
+            Err(err) => Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))?,
+        };
+
+        let decompressed = flate2::read::GzDecoder::new(data.as_slice());
+        tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| {
+            fs::remove_dir_all(package_dir).ok();
+            PackageError::MalformedArchive(Some(eco_format!("{err}")))
+        })
+    }
+}

crates/typst-kit/src/package_downloads/mod.rs

@@ -0,0 +1,203 @@
+//! This module provides the package downloader abstraction needed
+//! for remote package handling.
+//!
+//! # Content
+//!
+//! ## Traits
+//! The [PackageDownloader] trait provides the abstraction needed to implement
+//! multiple download method handlers.
+//! Each method must allow for a package download to the local filesystem and it should provide a
+//! method for downloading the repository index if it exists.
+//!
+//! The [Progress] trait allows for the implementation of a progress reporting struct.
+//!
+//! ## Module
+//! [http] contains functionality for making simple web requests with status reporting,
+//! useful for downloading packages from package registries.
+//! It is enabled by the `downloads_http` feature flag.
+//! Additionally the `vendor-openssl` can be used on operating systems other than macOS
+//! and Windows to vendor OpenSSL when building.
+//!
+//! [git] contains functionality for handling package downloads through git repositories.
+
+use ecow::{eco_format, EcoString};
+use std::collections::VecDeque;
+use std::fmt::Debug;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+use typst_library::diag::{PackageError, PackageResult};
+use typst_syntax::package::{PackageInfo, PackageSpec, VersionlessPackageSpec};
+
+/// The public namespace in the default Typst registry.
+pub const DEFAULT_NAMESPACE: &str = "preview";
+
+/*========BEGIN DOWNLOAD METHODS DECLARATION=========*/
+#[cfg(feature = "downloads_http")]
+pub mod http;
+
+#[cfg(feature = "downloads_git")]
+pub mod git;
+/*========END DOWNLOAD METHODS DECLARATION===========*/
+
+/// Trait abstraction for package a downloader.
+pub trait PackageDownloader: Debug + Sync + Send {
+    /// Download the repository index and returns the
+    /// list of PackageInfo elements contained in it.
+    fn download_index(
+        &self,
+        spec: &VersionlessPackageSpec,
+    ) -> Result<Vec<PackageInfo>, EcoString>;
+
+    /// Download a package from a remote repository/registry
+    /// and writes it in the file system cache directory
+    fn download(
+        &self,
+        spec: &PackageSpec,
+        package_dir: &Path,
+        progress: &mut dyn Progress,
+    ) -> PackageResult<()>;
+}
+
+/// The current state of an in progress or finished download.
+#[derive(Debug)]
+pub struct DownloadState {
+    /// The expected amount of bytes to download, `None` if the response header
+    /// was not set.
+    pub content_len: Option<usize>,
+    /// The total amount of downloaded bytes until now.
+    pub total_downloaded: usize,
+    /// A backlog of the amount of downloaded bytes each second.
+    pub bytes_per_second: VecDeque<usize>,
+    /// The download starting instant.
+    pub start_time: Instant,
+}
+
+/// Manages progress reporting for downloads.
+pub trait Progress {
+    /// Invoked when a download is started.
+    fn print_start(&mut self);
+
+    /// Invoked repeatedly while a download is ongoing.
+    fn print_progress(&mut self, state: &DownloadState);
+
+    /// Invoked when a download is finished.
+    fn print_finish(&mut self, state: &DownloadState);
+}
+
+/// The downloader object used for downloading packages
+#[derive(Debug)]
+pub struct Downloader {
+    ///List of all available downloaders which can be instantiated at runtime
+    http_downloader: Option<Box<dyn PackageDownloader>>,
+    git_downloader: Option<Box<dyn PackageDownloader>>,
+}
+
+impl Downloader {
+    /// Construct the Downloader object instantiating all the available methods.
+    /// The methods can be compile-time selected by features.
+    pub fn new(cert: Option<PathBuf>) -> Self {
+        Self {
+            http_downloader: Self::make_http_downloader(cert.clone()),
+            git_downloader: Self::make_git_downloader(cert),
+        }
+    }
+
+    /// Creation function for the HTTP(S) download method
+    fn make_http_downloader(cert: Option<PathBuf>) -> Option<Box<dyn PackageDownloader>> {
+        #[cfg(not(feature = "downloads_http"))]
+        {
+            None
+        }
+
+        #[cfg(feature = "downloads_http")]
+        {
+            match cert {
+                Some(cert_path) => Some(Box::new(http::HttpDownloader::with_path(
+                    http::HttpDownloader::default_user_agent(),
+                    cert_path,
+                ))),
+                None => Some(Box::new(http::HttpDownloader::new(
+                    http::HttpDownloader::default_user_agent(),
+                ))),
+            }
+        }
+    }
+
+    fn get_http_downloader(&self) -> Result<&dyn PackageDownloader, PackageError> {
+        let reference = self.http_downloader.as_ref().ok_or_else(|| {
+            PackageError::Other(Some(EcoString::from(
+                "Http downloader has not been initialized correctly",
+            )))
+        })?;
+        Ok(&**reference)
+    }
+
+    /// Creation function for the GIT clone method
+    fn make_git_downloader(_cert: Option<PathBuf>) -> Option<Box<dyn PackageDownloader>> {
+        #[cfg(not(feature = "downloads_git"))]
+        {
+            None
+        }
+
+        #[cfg(feature = "downloads_git")]
+        {
+            Some(Box::new(git::GitDownloader::new()))
+        }
+    }
+
+    fn get_git_downloader(&self) -> Result<&dyn PackageDownloader, PackageError> {
+        let reference = self.git_downloader.as_ref().ok_or_else(|| {
+            PackageError::Other(Some(EcoString::from(
+                "Http downloader has not been initialized correctly",
+            )))
+        })?;
+        Ok(&**reference)
+    }
+
+    /// Returns the correct downloader in function of the package namespace.
+    /// The remote location of a package is encoded in its namespace in the form
+    /// @<source type>:<source path>
+    ///
+    /// It's the downloader instance's job to parse the source path in any substructure.
+    ///
+    /// NOTE: Treating @preview as a special case of the https downloader.
+    fn get_downloader(&self, ns: &str) -> Result<&dyn PackageDownloader, PackageError> {
+        let download_type = ns.split(":").next();
+
+        match download_type {
+            #[cfg(feature = "downloads_http")]
+            Some("http") | Some("https") | Some("preview") => self.get_http_downloader(),
+
+            #[cfg(feature = "downloads_git")]
+            Some("git") => self.get_git_downloader(),
+
+            Some(dwld) => Err(PackageError::Other(Some(eco_format!(
+                "Unknown downloader type: {}",
+                dwld
+            )))),
+            None => Err(PackageError::Other(Some(EcoString::from(
+                "No downloader type specified",
+            )))),
+        }
+    }
+}
+
+impl PackageDownloader for Downloader {
+    fn download_index(
+        &self,
+        spec: &VersionlessPackageSpec,
+    ) -> Result<Vec<PackageInfo>, EcoString> {
+        let downloader = self.get_downloader(spec.namespace.as_str())?;
+        downloader.download_index(spec)
+    }
+
+    fn download(
+        &self,
+        spec: &PackageSpec,
+        package_dir: &Path,
+        progress: &mut dyn Progress,
+    ) -> PackageResult<()> {
+        let downloader = self.get_downloader(spec.namespace.as_str())?;
+        downloader.download(spec, package_dir, progress)
+    }
+}

crates/typst-syntax/src/package.rs

@@ -263,15 +263,37 @@ impl Display for VersionlessPackageSpec {
     }
 }
 
+fn is_namespace_valid(namespace: &str) -> bool {
+    if is_ident(namespace) {
+        //standard namespace
+        return true;
+    }
+
+    //if not ident, the namespace should be formed as @<package_remote_type>:<package_path>
+    let mut tokenized = namespace.splitn(2, ":");
+
+    //package type
+    let package_remote_type = tokenized.next();
+    if package_remote_type.is_none() || !is_ident(package_remote_type.unwrap()) {
+        return false;
+    }
+
+    //the package_path parsing is left to the downloader implementation
+    true
+}
+
 fn parse_namespace<'s>(s: &mut Scanner<'s>) -> Result<&'s str, EcoString> {
     if !s.eat_if('@') {
         Err("package specification must start with '@'")?;
     }
 
+    //todo: allow for multiple slashes in the by eating until last slash
     let namespace = s.eat_until('/');
     if namespace.is_empty() {
         Err("package specification is missing namespace")?;
-    } else if !is_ident(namespace) {
+    }
+
+    if !is_namespace_valid(namespace) {
         Err(eco_format!("`{namespace}` is not a valid package namespace"))?;
     }