Add support for URLs or paths in --nix-package-url and --extra-conf (#634)

* Add support for URLs or paths in --nix-package-url and --extra-conf

* fmt

* Into a mod with you, tests!
This commit is contained in:
Ana Hobden 2023-09-20 12:10:56 -07:00 committed by GitHub
parent 60e5fff623
commit abfde74d1f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 371 additions and 77 deletions

View file

@ -7,6 +7,7 @@ use tracing::{span, Span};
use crate::{
action::{Action, ActionDescription, ActionError, ActionErrorKind, ActionTag, StatefulAction},
parse_ssl_cert,
settings::UrlOrPath,
};
/**
@ -14,7 +15,7 @@ Fetch a URL to the given path
*/
#[derive(Debug, serde::Deserialize, serde::Serialize, Clone)]
pub struct FetchAndUnpackNix {
url: Url,
url_or_path: UrlOrPath,
dest: PathBuf,
proxy: Option<Url>,
ssl_cert_file: Option<PathBuf>,
@ -23,7 +24,7 @@ pub struct FetchAndUnpackNix {
impl FetchAndUnpackNix {
#[tracing::instrument(level = "debug", skip_all)]
pub async fn plan(
url: Url,
url_or_path: UrlOrPath,
dest: PathBuf,
proxy: Option<Url>,
ssl_cert_file: Option<PathBuf>,
@ -31,10 +32,12 @@ impl FetchAndUnpackNix {
// TODO(@hoverbear): Check URL exists?
// TODO(@hoverbear): Check tempdir exists
if let UrlOrPath::Url(url) = &url_or_path {
match url.scheme() {
"https" | "http" | "file" => (),
_ => return Err(Self::error(FetchUrlError::UnknownUrlScheme)),
};
_ => return Err(Self::error(ActionErrorKind::UnknownUrlScheme)),
}
}
if let Some(proxy) = &proxy {
match proxy.scheme() {
@ -48,7 +51,7 @@ impl FetchAndUnpackNix {
}
Ok(Self {
url,
url_or_path,
dest,
proxy,
ssl_cert_file,
@ -64,14 +67,14 @@ impl Action for FetchAndUnpackNix {
ActionTag("fetch_and_unpack_nix")
}
fn tracing_synopsis(&self) -> String {
format!("Fetch `{}` to `{}`", self.url, self.dest.display())
format!("Fetch `{}` to `{}`", self.url_or_path, self.dest.display())
}
fn tracing_span(&self) -> Span {
let span = span!(
tracing::Level::DEBUG,
"fetch_and_unpack_nix",
url = tracing::field::display(&self.url),
url_or_path = tracing::field::display(&self.url_or_path),
proxy = tracing::field::Empty,
ssl_cert_file = tracing::field::Empty,
dest = tracing::field::display(self.dest.display()),
@ -94,47 +97,60 @@ impl Action for FetchAndUnpackNix {
#[tracing::instrument(level = "debug", skip_all)]
async fn execute(&mut self) -> Result<(), ActionError> {
let bytes = match self.url.scheme() {
let bytes = match &self.url_or_path {
UrlOrPath::Url(url) => {
let bytes = match url.scheme() {
"https" | "http" => {
let mut buildable_client = reqwest::Client::builder();
if let Some(proxy) = &self.proxy {
buildable_client = buildable_client.proxy(
reqwest::Proxy::all(proxy.clone())
.map_err(FetchUrlError::Reqwest)
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?,
)
}
if let Some(ssl_cert_file) = &self.ssl_cert_file {
let ssl_cert = parse_ssl_cert(ssl_cert_file).await.map_err(Self::error)?;
let ssl_cert =
parse_ssl_cert(ssl_cert_file).await.map_err(Self::error)?;
buildable_client = buildable_client.add_root_certificate(ssl_cert);
}
let client = buildable_client
.build()
.map_err(FetchUrlError::Reqwest)
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?;
let req = client
.get(self.url.clone())
.get(url.clone())
.build()
.map_err(FetchUrlError::Reqwest)
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?;
let res = client
.execute(req)
.await
.map_err(FetchUrlError::Reqwest)
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?;
res.bytes()
.await
.map_err(FetchUrlError::Reqwest)
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?
},
"file" => {
let buf = tokio::fs::read(self.url.path())
let buf = tokio::fs::read(url.path())
.await
.map_err(|e| ActionErrorKind::Read(PathBuf::from(self.url.path()), e))
.map_err(|e| ActionErrorKind::Read(PathBuf::from(url.path()), e))
.map_err(Self::error)?;
Bytes::from(buf)
},
_ => return Err(Self::error(ActionErrorKind::UnknownUrlScheme)),
};
bytes
},
UrlOrPath::Path(path) => {
let buf = tokio::fs::read(path)
.await
.map_err(|e| ActionErrorKind::Read(PathBuf::from(path), e))
.map_err(Self::error)?;
Bytes::from(buf)
},
_ => return Err(Self::error(FetchUrlError::UnknownUrlScheme)),
};
// TODO(@Hoverbear): Pick directory
@ -167,16 +183,8 @@ impl Action for FetchAndUnpackNix {
#[non_exhaustive]
#[derive(Debug, thiserror::Error)]
pub enum FetchUrlError {
#[error("Request error")]
Reqwest(
#[from]
#[source]
reqwest::Error,
),
#[error("Unarchiving error")]
Unarchive(#[source] std::io::Error),
#[error("Unknown url scheme, `file://`, `https://` and `http://` supported")]
UnknownUrlScheme,
#[error("Unknown proxy scheme, `https://`, `socks5://`, and `http://` supported")]
UnknownProxyScheme,
}

View file

@ -43,6 +43,7 @@ impl ConfigureNix {
};
let place_nix_configuration = PlaceNixConfiguration::plan(
settings.nix_build_group_name.clone(),
settings.proxy.clone(),
settings.ssl_cert_file.clone(),
settings.extra_conf.clone(),
settings.force,

View file

@ -1,10 +1,13 @@
use tracing::{span, Span};
use url::Url;
use crate::action::base::create_or_insert_into_file::Position;
use crate::action::base::{CreateDirectory, CreateFile, CreateOrInsertIntoFile};
use crate::action::{
Action, ActionDescription, ActionError, ActionErrorKind, ActionTag, StatefulAction,
};
use crate::parse_ssl_cert;
use crate::settings::UrlOrPathOrString;
use std::path::{Path, PathBuf};
const NIX_CONF_FOLDER: &str = "/etc/nix";
@ -24,17 +27,70 @@ impl PlaceNixConfiguration {
#[tracing::instrument(level = "debug", skip_all)]
pub async fn plan(
nix_build_group_name: String,
proxy: Option<Url>,
ssl_cert_file: Option<PathBuf>,
extra_conf: Vec<String>,
extra_conf: Vec<UrlOrPathOrString>,
force: bool,
) -> Result<StatefulAction<Self>, ActionError> {
let create_directory = CreateDirectory::plan(NIX_CONF_FOLDER, None, None, 0o0755, force)
.await
.map_err(Self::error)?;
let mut extra_conf_text = vec![];
for extra in extra_conf {
let buf = match &extra {
UrlOrPathOrString::Url(url) => match url.scheme() {
"https" | "http" => {
let mut buildable_client = reqwest::Client::builder();
if let Some(proxy) = &proxy {
buildable_client = buildable_client.proxy(
reqwest::Proxy::all(proxy.clone())
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?,
)
}
if let Some(ssl_cert_file) = &ssl_cert_file {
let ssl_cert =
parse_ssl_cert(ssl_cert_file).await.map_err(Self::error)?;
buildable_client = buildable_client.add_root_certificate(ssl_cert);
}
let client = buildable_client
.build()
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?;
let req = client
.get(url.clone())
.build()
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?;
let res = client
.execute(req)
.await
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?;
res.text()
.await
.map_err(ActionErrorKind::Reqwest)
.map_err(Self::error)?
},
"file" => tokio::fs::read_to_string(url.path())
.await
.map_err(|e| ActionErrorKind::Read(PathBuf::from(url.path()), e))
.map_err(Self::error)?,
_ => return Err(Self::error(ActionErrorKind::UnknownUrlScheme)),
},
UrlOrPathOrString::Path(path) => tokio::fs::read_to_string(path)
.await
.map_err(|e| ActionErrorKind::Read(PathBuf::from(path), e))
.map_err(Self::error)?,
UrlOrPathOrString::String(string) => string.clone(),
};
extra_conf_text.push(buf)
}
let mut nix_conf_insert_settings = Vec::default();
nix_conf_insert_settings.push("include ./nix-installer-defaults.conf".into());
nix_conf_insert_settings.extend(extra_conf);
nix_conf_insert_settings.extend(extra_conf_text);
let nix_conf_insert_fragment = nix_conf_insert_settings.join("\n");
let mut defaults_conf_settings = vec![
@ -95,7 +151,7 @@ impl PlaceNixConfiguration {
// We only scan one include of depth -- we should make this any depth, make sure to guard for loops
if line.starts_with("include") || line.starts_with("!include") {
let allow_not_existing = line.starts_with("!");
let allow_not_existing = line.starts_with('!');
// Need to read it in if it exists for settings
let path = line
.trim_start_matches("include")

View file

@ -199,7 +199,7 @@ use std::{error::Error, process::Output};
use tokio::task::JoinError;
use tracing::Span;
use crate::{error::HasExpectedErrors, CertificateError};
use crate::{error::HasExpectedErrors, settings::UrlOrPathError, CertificateError};
use self::base::create_or_insert_into_file::Position;
@ -585,6 +585,16 @@ pub enum ActionErrorKind {
SystemdMissing,
#[error("`{command}` failed, message: {message}")]
DiskUtilInfoError { command: String, message: String },
#[error(transparent)]
UrlOrPathError(#[from] UrlOrPathError),
#[error("Request error")]
Reqwest(
#[from]
#[source]
reqwest::Error,
),
#[error("Unknown url scheme")]
UnknownUrlScheme,
}
impl ActionErrorKind {

View file

@ -1,9 +1,12 @@
/*! Configurable knobs and their related errors
*/
use std::{collections::HashMap, path::PathBuf};
use std::{collections::HashMap, fmt::Display, path::PathBuf, str::FromStr};
#[cfg(feature = "cli")]
use clap::ArgAction;
use clap::{
error::{ContextKind, ContextValue},
ArgAction,
};
use url::Url;
pub const SCRATCH_DIR: &str = "/nix/temp-install-dir";
@ -146,7 +149,7 @@ pub struct CommonSettings {
/// The Nix package URL
#[cfg_attr(
feature = "cli",
clap(long, env = "NIX_INSTALLER_NIX_PACKAGE_URL", global = true)
clap(long, env = "NIX_INSTALLER_NIX_PACKAGE_URL", global = true, value_parser = clap::value_parser!(UrlOrPath))
)]
#[cfg_attr(
all(target_os = "macos", target_arch = "x86_64", feature = "cli"),
@ -178,7 +181,7 @@ pub struct CommonSettings {
default_value = NIX_AARCH64_LINUX_URL,
)
)]
pub nix_package_url: Url,
pub nix_package_url: UrlOrPath,
/// The proxy to use (if any), valid proxy bases are `https://$URL`, `http://$URL` and `socks5://$URL`
#[cfg_attr(feature = "cli", clap(long, env = "NIX_INSTALLER_PROXY"))]
@ -190,7 +193,7 @@ pub struct CommonSettings {
/// Extra configuration lines for `/etc/nix.conf`
#[cfg_attr(feature = "cli", clap(long, action = ArgAction::Append, num_args = 0.., env = "NIX_INSTALLER_EXTRA_CONF", global = true))]
pub extra_conf: Vec<String>,
pub extra_conf: Vec<UrlOrPathOrString>,
/// If `nix-installer` should forcibly recreate files it finds existing
#[cfg_attr(
@ -384,6 +387,7 @@ impl CommonSettings {
Ok(map)
}
}
#[cfg(target_os = "linux")]
async fn linux_detect_systemd_started() -> bool {
use std::process::Stdio;
@ -516,6 +520,153 @@ pub enum InstallSettingsError {
),
#[error("No supported init system found")]
InitNotSupported,
#[error(transparent)]
UrlOrPath(#[from] UrlOrPathError),
}
#[derive(Debug, thiserror::Error)]
pub enum UrlOrPathError {
#[error("Error parsing URL `{0}`")]
Url(String, #[source] url::ParseError),
#[error("The specified path `{0}` does not exist")]
PathDoesNotExist(PathBuf),
#[error("Error fetching URL `{0}`")]
Reqwest(Url, #[source] reqwest::Error),
#[error("I/O error when accessing `{0}`")]
Io(PathBuf, #[source] std::io::Error),
}
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, Clone)]
pub enum UrlOrPath {
Url(Url),
Path(PathBuf),
}
impl Display for UrlOrPath {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
UrlOrPath::Url(url) => f.write_fmt(format_args!("{url}")),
UrlOrPath::Path(path) => f.write_fmt(format_args!("{}", path.display())),
}
}
}
impl FromStr for UrlOrPath {
type Err = UrlOrPathError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match Url::parse(s) {
Ok(url) => Ok(UrlOrPath::Url(url)),
Err(url::ParseError::RelativeUrlWithoutBase) => {
// This is most likely a relative path (`./boop` or `boop`)
// or an absolute path (`/boop`)
//
// So we'll see if such a path exists, and if so, use it
let path = PathBuf::from(s);
if path.exists() {
Ok(UrlOrPath::Path(path))
} else {
Err(UrlOrPathError::PathDoesNotExist(path))
}
},
Err(e) => Err(UrlOrPathError::Url(s.to_string(), e)),
}
}
}
#[cfg(feature = "cli")]
impl clap::builder::TypedValueParser for UrlOrPath {
type Value = UrlOrPath;
fn parse_ref(
&self,
cmd: &clap::Command,
_arg: Option<&clap::Arg>,
value: &std::ffi::OsStr,
) -> Result<Self::Value, clap::Error> {
let value_str = value.to_str().ok_or_else(|| {
let mut err = clap::Error::new(clap::error::ErrorKind::InvalidValue);
err.insert(
ContextKind::InvalidValue,
ContextValue::String(format!("`{value:?}` not a UTF-8 string")),
);
err
})?;
match UrlOrPath::from_str(value_str) {
Ok(v) => Ok(v),
Err(from_str_error) => {
let mut err = clap::Error::new(clap::error::ErrorKind::InvalidValue).with_cmd(cmd);
err.insert(
clap::error::ContextKind::Custom,
clap::error::ContextValue::String(from_str_error.to_string()),
);
Err(err)
},
}
}
}
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, Clone)]
pub enum UrlOrPathOrString {
Url(Url),
Path(PathBuf),
String(String),
}
impl FromStr for UrlOrPathOrString {
type Err = url::ParseError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match Url::parse(s) {
Ok(url) => Ok(UrlOrPathOrString::Url(url)),
Err(url::ParseError::RelativeUrlWithoutBase) => {
// This is most likely a relative path (`./boop` or `boop`)
// or an absolute path (`/boop`)
//
// So we'll see if such a path exists, and if so, use it
let path = PathBuf::from(s);
if path.exists() {
Ok(UrlOrPathOrString::Path(path))
} else {
// The path doesn't exist, so the user is providing us with a string
Ok(UrlOrPathOrString::String(s.into()))
}
},
Err(e) => Err(e),
}
}
}
#[cfg(feature = "cli")]
impl clap::builder::TypedValueParser for UrlOrPathOrString {
type Value = UrlOrPathOrString;
fn parse_ref(
&self,
cmd: &clap::Command,
_arg: Option<&clap::Arg>,
value: &std::ffi::OsStr,
) -> Result<Self::Value, clap::Error> {
let value_str = value.to_str().ok_or_else(|| {
let mut err = clap::Error::new(clap::error::ErrorKind::InvalidValue);
err.insert(
ContextKind::InvalidValue,
ContextValue::String(format!("`{value:?}` not a UTF-8 string")),
);
err
})?;
match UrlOrPathOrString::from_str(value_str) {
Ok(v) => Ok(v),
Err(from_str_error) => {
let mut err = clap::Error::new(clap::error::ErrorKind::InvalidValue).with_cmd(cmd);
err.insert(
clap::error::ContextKind::Custom,
clap::error::ContextValue::String(from_str_error.to_string()),
);
Err(err)
},
}
}
}
#[cfg(feature = "diagnostics")]
@ -525,3 +676,48 @@ impl crate::diagnostics::ErrorDiagnostic for InstallSettingsError {
static_str.to_string()
}
}
#[cfg(test)]
mod tests {
use super::{FromStr, PathBuf, Url, UrlOrPath, UrlOrPathOrString};
#[test]
fn url_or_path_or_string_parses() -> Result<(), Box<dyn std::error::Error>> {
assert_eq!(
UrlOrPathOrString::from_str("https://boop.bleat")?,
UrlOrPathOrString::Url(Url::from_str("https://boop.bleat")?),
);
assert_eq!(
UrlOrPathOrString::from_str("file:///boop/bleat")?,
UrlOrPathOrString::Url(Url::from_str("file:///boop/bleat")?),
);
// The file *must* exist!
assert_eq!(
UrlOrPathOrString::from_str(file!())?,
UrlOrPathOrString::Path(PathBuf::from_str(file!())?),
);
assert_eq!(
UrlOrPathOrString::from_str("Boop")?,
UrlOrPathOrString::String(String::from("Boop")),
);
Ok(())
}
#[test]
fn url_or_path_parses() -> Result<(), Box<dyn std::error::Error>> {
assert_eq!(
UrlOrPath::from_str("https://boop.bleat")?,
UrlOrPath::Url(Url::from_str("https://boop.bleat")?),
);
assert_eq!(
UrlOrPath::from_str("file:///boop/bleat")?,
UrlOrPath::Url(Url::from_str("file:///boop/bleat")?),
);
// The file *must* exist!
assert_eq!(
UrlOrPath::from_str(file!())?,
UrlOrPath::Path(PathBuf::from_str(file!())?),
);
Ok(())
}
}

View file

@ -18,7 +18,9 @@
"action": "provision_nix",
"fetch_nix": {
"action": {
"url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz",
"url_or_path": {
"Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz"
},
"dest": "/nix/temp-install-dir",
"proxy": null,
"ssl_cert_file": null
@ -243,14 +245,14 @@
"create_directories": [
{
"action": {
"path": "/etc/zsh",
"path": "/etc/fish/conf.d",
"user": null,
"group": null,
"mode": 493,
"is_mountpoint": false,
"force_prune_on_revert": false
},
"state": "Uncompleted"
"state": "Completed"
},
{
"action": {
@ -320,6 +322,17 @@
},
"state": "Uncompleted"
},
{
"action": {
"path": "/etc/fish/conf.d/nix.fish",
"user": null,
"group": null,
"mode": 420,
"buf": "\n# Nix\nif test -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.fish'\n . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.fish'\nend\n# End Nix\n\n",
"position": "Beginning"
},
"state": "Uncompleted"
},
{
"action": {
"path": "/usr/share/fish/vendor_conf.d/nix.fish",
@ -353,19 +366,19 @@
"path": "/etc/nix/nix.conf",
"user": null,
"group": null,
"mode": null,
"buf": "!include ./nix-installer-defaults.conf",
"mode": 493,
"buf": "include ./nix-installer-defaults.conf\n",
"position": "Beginning"
},
"state": "Uncompleted"
},
"create_defaults_conf": {
"action": {
"path": "/etc/nix/defaults.conf",
"path": "/etc/nix/nix-installer-defaults.conf",
"user": null,
"group": null,
"mode": null,
"buf": "build-users-group = nixbld\nexperimental-features = nix-command flakes auto-allocate-uids\nbash-prompt-prefix = (nix:$name)\\040\nextra-nix-path = nixpkgs=flake:nixpkgs\nauto-optimise-store = true\nauto-allocate-uids = true",
"mode": 493,
"buf": "build-users-group = nixbld\nexperimental-features = nix-command flakes auto-allocate-uids\nbash-prompt-prefix = (nix:$name)\\040\nextra-nix-path = nixpkgs=flake:nixpkgs\nmax-jobs = auto\nauto-optimise-store = true\nauto-allocate-uids = true\n",
"replace": true
},
"state": "Uncompleted"
@ -386,7 +399,7 @@
"is_mountpoint": false,
"force_prune_on_revert": false
},
"state": "Completed"
"state": "Uncompleted"
},
{
"action": {
@ -413,7 +426,9 @@
"nix_build_user_prefix": "nixbld",
"nix_build_user_count": 0,
"nix_build_user_id_base": 30000,
"nix_package_url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz",
"nix_package_url": {
"Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz"
},
"proxy": null,
"ssl_cert_file": null,
"extra_conf": [],

View file

@ -68,7 +68,9 @@
"action": "provision_nix",
"fetch_nix": {
"action": {
"url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz",
"url_or_path": {
"Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz"
},
"dest": "/nix/temp-install-dir",
"proxy": null,
"ssl_cert_file": null
@ -443,7 +445,9 @@
"nix_build_user_prefix": "nixbld",
"nix_build_user_count": 0,
"nix_build_user_id_base": 30000,
"nix_package_url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz",
"nix_package_url": {
"Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz"
},
"proxy": null,
"ssl_cert_file": null,
"extra_conf": [],

View file

@ -88,7 +88,9 @@
"action": "provision_nix",
"fetch_nix": {
"action": {
"url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-darwin.tar.xz",
"url_or_path": {
"Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-darwin.tar.xz"
},
"dest": "/nix/temp-install-dir",
"proxy": null,
"ssl_cert_file": null
@ -1090,7 +1092,9 @@
"nix_build_user_prefix": "_nixbld",
"nix_build_user_count": 32,
"nix_build_user_id_base": 300,
"nix_package_url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-darwin.tar.xz",
"nix_package_url": {
"Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-darwin.tar.xz"
},
"proxy": null,
"ssl_cert_file": null,
"extra_conf": [],