Migrate to lix

Merge pull request #146 from zhaofengli/install-ci-hotfix
ci-installer: Remove drvPath from fake derivation
2024-07-19 21:20:46 +02:00 · 2024-07-09 12:27:54 -04:00 · 2024-07-09 12:26:18 -04:00 · 2024-07-09 08:12:19 -04:00 · 2024-07-09 07:56:03 -04:00 · 2024-06-08 22:05:37 -05:00
28 changed files with 1372 additions and 1062 deletions
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
--- a/.editorconfig
+++ b/.editorconfig
@ -13,7 +13,7 @@ charset = utf-8
 # Rust
 [*.rs]
 indent_style = space
-indent_size = 2
+indent_size = 4

 # Misc
 [*.{yaml,yml,md,nix}]
--- a/.github/install-attic-ci.sh
+++ b/.github/install-attic-ci.sh
@ -25,7 +25,6 @@ let
          value = common // {
            inherit outputName;
            outPath = maybeStorePath (builtins.getAttr outputName outputs);
-            drvPath = maybeStorePath (builtins.getAttr outputName outputs);
          };
        };
      outputsList = map outputToAttrListElement outputNames;
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -11,7 +11,7 @@ jobs:
      matrix:
        os:
          - ubuntu-latest
-          - macos-11
+          - macos-latest
    runs-on: ${{ matrix.os }}
    permissions:
      contents: read
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@ -6,7 +6,7 @@ publish = false

 [dependencies]
 async-stream = { version = "0.3.5", optional = true }
-base64 = "0.21.2"
+base64 = "0.22.1"
 bytes = "1.4.0"
 displaydoc = "0.2.4"
 digest = "0.10.7"
@ -53,7 +53,7 @@ default = [ "nix_store", "tokio" ]
 # Native libnixstore bindings.
 #
 # When disabled, the native Rust portions of nix_store can still be used.
-nix_store = [ "dep:cxx", "dep:cxx-build" ]
+nix_store = [ "dep:cxx", "dep:cxx-build", "tokio/rt" ]

 # Tokio.
 #
--- a/attic/build.rs
+++ b/attic/build.rs
@ -9,36 +9,28 @@ fn main() {

 #[cfg(feature = "nix_store")]
 fn build_bridge() {
-    // Temporary workaround for issue in <https://github.com/NixOS/nix/pull/8484>
-    let hacky_include = {
-        let dir = tempfile::tempdir().expect("Failed to create temporary directory for workaround");
-        std::fs::write(dir.path().join("uds-remote-store.md"), "\"\"").unwrap();
-        dir
-    };
+    let libstore = pkg_config::Config::new()
+        .probe("lix-store")
+        .unwrap();
+
+    let libmain = pkg_config::Config::new()
+        .probe("lix-main")
+        .unwrap();
+
+    let libutil = pkg_config::Config::new()
+        .probe("lix-util")
+        .unwrap();

    cxx_build::bridge("src/nix_store/bindings/mod.rs")
        .file("src/nix_store/bindings/nix.cpp")
        .flag("-std=c++2a")
        .flag("-O2")
        .flag("-include")
-        .flag("nix/config.h")
-        .flag("-idirafter")
-        .flag(hacky_include.path().to_str().unwrap())
-        // In Nix 2.19+, nix/args/root.hh depends on being able to #include "root.hh" (which is in its parent directory), for some reason
-        .flag("-I")
-        .flag(concat!(env!("NIX_INCLUDE_PATH"), "/nix"))
+        .flag("lix/config.h")
+        .includes(&libmain.include_paths)
+        .includes(&libutil.include_paths)
+        .includes(&libstore.include_paths)
        .compile("nixbinding");

    println!("cargo:rerun-if-changed=src/nix_store/bindings");
-
-    // the -l flags must be after -lnixbinding
-    pkg_config::Config::new()
-        .atleast_version("2.4")
-        .probe("nix-store")
-        .unwrap();
-
-    pkg_config::Config::new()
-        .atleast_version("2.4")
-        .probe("nix-main")
-        .unwrap();
 }
--- a/attic/src/nix_store/bindings/nix.cpp
+++ b/attic/src/nix_store/bindings/nix.cpp
@ -140,7 +140,7 @@ void CNixStore::nar_from_path(RVec<unsigned char> base_name, RBox<AsyncWriteSend
 	nix::StorePath store_path(sv);

 	// exceptions will be thrown into Rust
-	this->store->narFromPath(store_path, sink);
+	sink << this->store->narFromPath(store_path);
 	sink.eof();
 }

--- a/attic/src/nix_store/bindings/nix.hpp
+++ b/attic/src/nix_store/bindings/nix.hpp
@ -13,14 +13,14 @@
 #include <memory>
 #include <mutex>
 #include <set>
-#include <nix/store-api.hh>
-#include <nix/local-store.hh>
-#include <nix/remote-store.hh>
-#include <nix/uds-remote-store.hh>
-#include <nix/hash.hh>
-#include <nix/path.hh>
-#include <nix/serialise.hh>
-#include <nix/shared.hh>
+#include <libstore/store-api.hh>
+#include <libstore/local-store.hh>
+#include <libstore/remote-store.hh>
+#include <libstore/uds-remote-store.hh>
+#include <libutil/hash.hh>
+#include <libstore/path.hh>
+#include <libutil/serialise.hh>
+#include <libmain/shared.hh>
 #include <rust/cxx.h>

 template<class T> using RVec = rust::Vec<T>;
--- a/ci-installer.nix
+++ b/ci-installer.nix
@ -31,7 +31,6 @@ let
              value = common // {
                inherit outputName;
                outPath = maybeStorePath (builtins.getAttr outputName outputs);
-                drvPath = maybeStorePath (builtins.getAttr outputName outputs);
              };
            };
          outputsList = map outputToAttrListElement outputNames;
--- a/client/Cargo.toml
+++ b/client/Cargo.toml
@ -12,7 +12,7 @@ path = "src/main.rs"
 attic = { path = "../attic" }

 anyhow = "1.0.71"
-async-channel = "1.8.0"
+async-channel = "2.3.1"
 bytes = "1.4.0"
 clap = { version = "4.3", features = ["derive"] }
 clap_complete = "4.3.0"
@ -26,7 +26,7 @@ indicatif = "0.17.3"
 lazy_static = "1.4.0"
 notify = { version = "6.0.0", default-features = false, features = ["macos_kqueue"] }
 regex = "1.8.3"
-reqwest = { version = "0.11.18", default-features = false, features = ["json", "rustls-tls", "rustls-tls-native-roots", "stream"] }
+reqwest = { version = "0.12.4", default-features = false, features = ["json", "rustls-tls", "rustls-tls-native-roots", "stream"] }
 serde = { version = "1.0.163", features = ["derive"] }
 serde_json = "1.0.96"
 toml = "0.8.8"
--- a/client/src/config.rs
+++ b/client/src/config.rs
@ -81,6 +81,7 @@ impl ServerTokenConfig {
        match self {
            ServerTokenConfig::Raw { token } => Ok(token.clone()),
            ServerTokenConfig::File { token_file } => Ok(read_to_string(token_file)
+                .map(|t| t.trim().to_string())
                .with_context(|| format!("Failed to read token from {token_file}"))?),
        }
    }
--- a/crane.nix
+++ b/crane.nix
@ -62,9 +62,6 @@ let

    ATTIC_DISTRIBUTOR = "attic";

-    # See comment in `attic/build.rs`
-    NIX_INCLUDE_PATH = "${lib.getDev nix}/include";
-
    # See comment in `attic-tests`
    doCheck = false;

@ -139,9 +136,6 @@ let
    checkPhaseCargoCommand = "cargoWithProfile test --no-run --message-format=json >cargo-test.json";
    doInstallCargoArtifacts = false;

-    # See comment in `attic/build.rs`
-    NIX_INCLUDE_PATH = "${lib.getDev nix}/include";
-
    installPhase = ''
      runHook preInstall

--- a/flake.lock
+++ b/flake.lock
@ -7,11 +7,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1702918879,
-        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "lastModified": 1721322122,
+        "narHash": "sha256-a0G1NvyXGzdwgu6e1HQpmK5R5yLsfxeBe07nNDyYd+g=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "rev": "8a68b987c476a33e90f203f0927614a75c3f47ea",
        "type": "github"
      },
      "original": {
@ -23,11 +23,11 @@
    "flake-compat": {
      "flake": false,
      "locked": {
-        "lastModified": 1673956053,
-        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
        "owner": "edolstra",
        "repo": "flake-compat",
-        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
        "type": "github"
      },
      "original": {
@ -37,12 +37,15 @@
      }
    },
    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
      "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
        "owner": "numtide",
        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
        "type": "github"
      },
      "original": {
@ -51,13 +54,87 @@
        "type": "github"
      }
    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flakey-profile": {
+      "locked": {
+        "lastModified": 1712898590,
+        "narHash": "sha256-FhGIEU93VHAChKEXx905TSiPZKga69bWl1VB37FK//I=",
+        "owner": "lf-",
+        "repo": "flakey-profile",
+        "rev": "243c903fd8eadc0f63d205665a92d4df91d42d9d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "lf-",
+        "repo": "flakey-profile",
+        "type": "github"
+      }
+    },
+    "lix": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1721371213,
+        "narHash": "sha256-7SdrlNe5DBlK5uLBhPPxVRWI50N1PFz3zMBeDYiX0Qs=",
+        "ref": "refs/heads/main",
+        "rev": "aba5f19680b2f4c29d7ce2ff5e2a89128c1cb26d",
+        "revCount": 15985,
+        "type": "git",
+        "url": "ssh://git@git.lix.systems/lix-project/lix"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://git@git.lix.systems/lix-project/lix"
+      }
+    },
+    "lix-module": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "flakey-profile": "flakey-profile",
+        "lix": [
+          "lix"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1720695775,
+        "narHash": "sha256-8Oqzl9QPjEe/n8y0R2tC6+2v/H6xBgABHXOJwxmnBg0=",
+        "ref": "refs/heads/main",
+        "rev": "d70318fb946a0e720dfdd1fb10b0645c14e2a02a",
+        "revCount": 94,
+        "type": "git",
+        "url": "ssh://git@git.lix.systems/lix-project/nixos-module"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://git@git.lix.systems/lix-project/nixos-module"
+      }
+    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1711401922,
-        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
+        "lastModified": 1721373214,
+        "narHash": "sha256-crpGeGQGFlnCsMyCE5eheyjzo3xo03o1FXJ2sAbm7No=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
+        "rev": "af9c15bc7a314c226d7d5d85e159f7a73e8d9fae",
        "type": "github"
      },
      "original": {
@ -67,29 +144,44 @@
        "type": "github"
      }
    },
-    "nixpkgs-stable": {
-      "locked": {
-        "lastModified": 1711460390,
-        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-23.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
    "root": {
      "inputs": {
        "crane": "crane",
        "flake-compat": "flake-compat",
        "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-stable": "nixpkgs-stable"
+        "lix": "lix",
+        "lix-module": "lix-module",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
      }
    }
  },
--- a/flake.nix
+++ b/flake.nix
@ -3,7 +3,15 @@

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
-    nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-23.11";
+    lix = {
+      url = "git+ssh://git@git.lix.systems/lix-project/lix";
+      flake = false;
+    };
+    lix-module = {
+      url = "git+ssh://git@git.lix.systems/lix-project/nixos-module";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.lix.follows = "lix";
+    };
    flake-utils.url = "github:numtide/flake-utils";

    crane = {
@ -17,7 +25,7 @@
    };
  };

-  outputs = { self, nixpkgs, nixpkgs-stable, flake-utils, crane, ... }: let
+  outputs = { self, nixpkgs, lix-module, flake-utils, crane, ... }: let
    supportedSystems = flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ];

    makeCranePkgs = pkgs: let
@ -26,16 +34,10 @@
  in flake-utils.lib.eachSystem supportedSystems (system: let
    pkgs = import nixpkgs {
      inherit system;
-      overlays = [];
+      overlays = [lix-module.overlays.default];
    };
    cranePkgs = makeCranePkgs pkgs;

-    pkgsStable = import nixpkgs-stable {
-      inherit system;
-      overlays = [];
-    };
-    cranePkgsStable = makeCranePkgs pkgsStable;
-
    inherit (pkgs) lib;
  in rec {
    packages = {
@ -55,17 +57,7 @@
    } // (lib.optionalAttrs (system != "x86_64-darwin") {
      # Unfortunately, x86_64-darwin fails to evaluate static builds
      # TODO: Make this work with Crane
-      attic-static = (pkgs.pkgsStatic.callPackage ./package.nix {
-        nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
-          patches = (old.patches or []) ++ [
-            # To be submitted
-            (pkgs.fetchpatch {
-              url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
-              hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
-            })
-          ];
-        });
-      }).overrideAttrs (old: {
+      attic-static = (pkgs.pkgsStatic.callPackage ./package.nix { }).overrideAttrs (old: {
        nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
          pkgs.nukeReferences
        ];
@ -127,12 +119,8 @@
          linuxPackages.perf
        ]);

-        NIX_PATH = "nixpkgs=${pkgs.path}";
        RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library";

-        # See comment in `attic/build.rs`
-        NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include";
-
        ATTIC_DISTRIBUTOR = "dev";
      };

@ -163,8 +151,7 @@
        flake = self;
      };
      unstableTests = makeIntegrationTests pkgs;
-      stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable);
-    in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests);
+    in lib.optionalAttrs pkgs.stdenv.isLinux unstableTests;
  }) // {
    overlays = {
      default = final: prev: let
--- a/integration-tests/default.nix
+++ b/integration-tests/default.nix
@ -33,6 +33,6 @@ let
        }
      ];
    };
-  }) (lib.cartesianProductOfSets matrix));
+  }) (lib.cartesianProduct matrix));
 in {
 } // basicTests
--- a/package.nix
+++ b/package.nix
@ -49,9 +49,6 @@ in rustPlatform.buildRustPackage rec {

  ATTIC_DISTRIBUTOR = "attic";

-  # See comment in `attic/build.rs`
-  NIX_INCLUDE_PATH = "${lib.getDev nix}/include";
-
  # Recursive Nix is not stable yet
  doCheck = false;

--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@ -25,11 +25,11 @@ attic-token = { path = "../token" }
 anyhow = "1.0.71"
 async-stream = "0.3.5"
 async-trait = "0.1.68"
-aws-config = "0.57.1"
-aws-sdk-s3 = "0.35.0"
-axum = "0.6.18"
-axum-macros = "0.3.7"
-base64 = "0.21.2"
+aws-config = "1.5.0"
+aws-sdk-s3 = "1.32.0"
+axum = "0.7.5"
+axum-macros = "0.4.1"
+base64 = "0.22.1"
 bytes = "1.4.0"
 chrono = "0.4.24"
 clap = { version = "4.3", features = ["derive"] }
@ -40,6 +40,7 @@ enum-as-inner = "0.6.0"
 fastcdc = "3.0.3"
 futures = "0.3.28"
 hex = "0.4.3"
+http-body-util = "0.1.1"
 humantime = "2.1.0"
 humantime-serde = "1.1.1"
 itoa = "=1.0.5"
@ -53,7 +54,7 @@ serde_json = "1.0.96"
 serde_with = "3.0.0"
 tokio-util = { version = "0.7.8", features = [ "io" ] }
 toml = "0.8.8"
-tower-http = { version = "0.4.0", features = [ "catch-panic", "trace" ] }
+tower-http = { version = "0.5.2", features = [ "catch-panic", "trace" ] }
 tracing = "0.1.37"
 tracing-error = "0.2.0"
 tracing-subscriber = { version = "0.3.17", features = [ "json" ] }
--- a/server/src/access/http.rs
+++ b/server/src/access/http.rs
@ -1,6 +1,6 @@
 //! HTTP middlewares for access control.

-use axum::{http::Request, middleware::Next, response::Response};
+use axum::{extract::Request, middleware::Next, response::Response};
 use sea_orm::DatabaseConnection;
 use tokio::sync::OnceCell;

@ -93,7 +93,7 @@ impl AuthState {
 }

 /// Performs auth.
-pub async fn apply_auth<B>(req: Request<B>, next: Next<B>) -> Response {
+pub async fn apply_auth(req: Request, next: Next) -> Response {
    let token: Option<Token> = req
        .headers()
        .get("Authorization")
--- a/server/src/api/binary_cache.rs
+++ b/server/src/api/binary_cache.rs
@ -10,7 +10,7 @@ use std::path::PathBuf;
 use std::sync::Arc;

 use axum::{
-    body::StreamBody,
+    body::Body,
    extract::{Extension, Path},
    http::StatusCode,
    response::{IntoResponse, Redirect, Response},
@ -18,6 +18,7 @@ use axum::{
    Router,
 };
 use futures::stream::BoxStream;
+use http_body_util::BodyExt;
 use serde::Serialize;
 use tokio_util::io::ReaderStream;
 use tracing::instrument;
@ -217,7 +218,11 @@ async fn get_nar(
            Download::Url(url) => Ok(Redirect::temporary(&url).into_response()),
            Download::AsyncRead(stream) => {
                let stream = ReaderStream::new(stream);
-                let body = StreamBody::new(stream);
+                let body = Body::from_stream(stream).map_err(|e| {
+                    tracing::error!("Stream error: {e}");
+                    e
+                }).into_inner();
+
                Ok(body.into_response())
            }
        }
@ -250,7 +255,11 @@ async fn get_nar(
        // TODO: Make num_prefetch configurable
        // The ideal size depends on the average chunk size
        let merged = merge_chunks(chunks, streamer, storage, 2);
-        let body = StreamBody::new(merged);
+        let body = Body::from_stream(merged).map_err(|e| {
+            tracing::error!("Stream error: {e}");
+            e
+        }).into_inner();
+
        Ok(body.into_response())
    }
 }
--- a/server/src/api/v1/upload_path.rs
+++ b/server/src/api/v1/upload_path.rs
@ -8,7 +8,8 @@ use anyhow::anyhow;
 use async_compression::tokio::bufread::{BrotliEncoder, XzEncoder, ZstdEncoder};
 use async_compression::Level as CompressionLevel;
 use axum::{
-    extract::{BodyStream, Extension, Json},
+    body::Body,
+    extract::{Extension, Json},
    http::HeaderMap,
 };
 use bytes::{Bytes, BytesMut};
@ -120,8 +121,9 @@ pub(crate) async fn upload_path(
    Extension(state): Extension<State>,
    Extension(req_state): Extension<RequestState>,
    headers: HeaderMap,
-    stream: BodyStream,
+    body: Body,
 ) -> ServerResult<Json<UploadPathResult>> {
+    let stream = body.into_data_stream();
    let mut stream = StreamReader::new(
        stream.map(|r| r.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))),
    );
--- a/server/src/gc.rs
+++ b/server/src/gc.rs
@ -158,6 +158,15 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> {
    let db = state.database().await?;
    let storage = state.storage().await?;

+    let orphan_chunk_limit = match db.get_database_backend() {
+        // Arbitrarily chosen sensible value since there's no good default to choose from for MySQL 
+        sea_orm::DatabaseBackend::MySql    => 1000,
+        // Panic limit set by sqlx for postgresql: https://github.com/launchbadge/sqlx/issues/671#issuecomment-687043510
+        sea_orm::DatabaseBackend::Postgres => u64::from(u16::MAX),
+        // Default statement limit imposed by sqlite: https://www.sqlite.org/limits.html#max_variable_number 
+        sea_orm::DatabaseBackend::Sqlite   => 500,
+    };
+
    // find all orphan chunks...
    let orphan_chunk_ids = Query::select()
        .from(Chunk)
@ -190,6 +199,7 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> {

    let orphan_chunks: Vec<chunk::Model> = Chunk::find()
        .filter(chunk::Column::State.eq(ChunkState::Deleted))
+        .limit(orphan_chunk_limit)
        .all(db)
        .await?;

--- a/server/src/lib.rs
+++ b/server/src/lib.rs
@ -26,6 +26,7 @@ pub mod nix_manifest;
 pub mod oobe;
 mod storage;

+use std::future::IntoFuture;
 use std::net::SocketAddr;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
@ -38,6 +39,7 @@ use axum::{
    Router,
 };
 use sea_orm::{query::Statement, ConnectionTrait, Database, DatabaseConnection};
+use tokio::net::TcpListener;
 use tokio::sync::OnceCell;
 use tokio::time;
 use tower_http::catch_panic::CatchPanicLayer;
@ -105,9 +107,28 @@ impl StateInner {
    async fn database(&self) -> ServerResult<&DatabaseConnection> {
        self.database
            .get_or_try_init(|| async {
-                Database::connect(&self.config.database.url)
+                let db = Database::connect(&self.config.database.url)
                    .await
-                    .map_err(ServerError::database_error)
+                    .map_err(ServerError::database_error);
+                if let Ok(DatabaseConnection::SqlxSqlitePoolConnection(ref conn)) = db {
+                    // execute some sqlite-specific performance optimizations
+                    // see https://phiresky.github.io/blog/2020/sqlite-performance-tuning/ for
+                    // more details
+                    // intentionally ignore errors from this: this is purely for performance,
+                    // not for correctness, so we can live without this
+                    _ = conn
+                        .execute_unprepared(
+                            "
+                        pragma journal_mode=WAL;
+                        pragma synchronous=normal;
+                        pragma temp_store=memory;
+                        pragma mmap_size = 30000000000;
+                        ",
+                        )
+                        .await;
+                }
+
+                db
            })
            .await
    }
@ -221,14 +242,13 @@ pub async fn run_api_server(cli_listen: Option<SocketAddr>, config: Config) -> R

    eprintln!("Listening on {:?}...", listen);

-    let (server_ret, _) = tokio::join!(
-        axum::Server::bind(&listen).serve(rest.into_make_service()),
-        async {
-            if state.config.database.heartbeat {
-                let _ = state.run_db_heartbeat().await;
-            }
-        },
-    );
+    let listener = TcpListener::bind(&listen).await?;
+
+    let (server_ret, _) = tokio::join!(axum::serve(listener, rest).into_future(), async {
+        if state.config.database.heartbeat {
+            let _ = state.run_db_heartbeat().await;
+        }
+    },);

    server_ret?;

--- a/server/src/middleware.rs
+++ b/server/src/middleware.rs
@ -3,8 +3,8 @@ use std::sync::Arc;

 use anyhow::anyhow;
 use axum::{
-    extract::{Extension, Host},
-    http::{HeaderValue, Request},
+    extract::{Extension, Host, Request},
+    http::HeaderValue,
    middleware::Next,
    response::Response,
 };
@ -14,11 +14,11 @@ use crate::error::{ErrorKind, ServerResult};
 use attic::api::binary_cache::ATTIC_CACHE_VISIBILITY;

 /// Initializes per-request state.
-pub async fn init_request_state<B>(
+pub async fn init_request_state(
    Extension(state): Extension<State>,
    Host(host): Host,
-    mut req: Request<B>,
-    next: Next<B>,
+    mut req: Request,
+    next: Next,
 ) -> Response {
    // X-Forwarded-Proto is an untrusted header
    let client_claims_https =
@ -45,11 +45,11 @@ pub async fn init_request_state<B>(
 ///
 /// We also require that all request have a Host header in
 /// the first place.
-pub async fn restrict_host<B>(
+pub async fn restrict_host(
    Extension(state): Extension<State>,
    Host(host): Host,
-    req: Request<B>,
-    next: Next<B>,
+    req: Request,
+    next: Next,
 ) -> ServerResult<Response> {
    let allowed_hosts = &state.config.allowed_hosts;

@ -61,10 +61,10 @@ pub async fn restrict_host<B>(
 }

 /// Sets the `X-Attic-Cache-Visibility` header in responses.
-pub(crate) async fn set_visibility_header<B>(
+pub(crate) async fn set_visibility_header(
    Extension(req_state): Extension<RequestState>,
-    req: Request<B>,
-    next: Next<B>,
+    req: Request,
+    next: Next,
 ) -> ServerResult<Response> {
    let mut response = next.run(req).await;

--- a/server/src/storage/s3.rs
+++ b/server/src/storage/s3.rs
@ -3,6 +3,7 @@
 use std::time::Duration;

 use async_trait::async_trait;
+use aws_config::BehaviorVersion;
 use aws_sdk_s3::{
    config::Builder as S3ConfigBuilder,
    config::{Credentials, Region},
@ -91,7 +92,7 @@ impl S3Backend {
    }

    async fn config_builder(config: &S3StorageConfig) -> ServerResult<S3ConfigBuilder> {
-        let shared_config = aws_config::load_from_env().await;
+        let shared_config = aws_config::load_defaults(BehaviorVersion::v2024_03_28()).await;
        let mut builder = S3ConfigBuilder::from(&shared_config);

        if let Some(credentials) = &config.credentials {
--- a/token/Cargo.toml
+++ b/token/Cargo.toml
@ -8,9 +8,10 @@ edition = "2021"
 [dependencies]
 attic = { path = "../attic", default-features = false }

-base64 = "0.21.2"
+base64 = "0.22.1"
 chrono = "0.4.24"
 displaydoc = "0.2.4"
+indexmap = { version = "2.2.6", features = ["serde"] }
 jwt-simple = "0.11.5"
 lazy_static = "1.4.0"
 regex = "1.8.3"
--- a/token/src/lib.rs
+++ b/token/src/lib.rs
@ -83,12 +83,12 @@ pub mod util;
 #[cfg(test)]
 mod tests;

-use std::collections::HashMap;
 use std::error::Error as StdError;

 use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine};
 use chrono::{DateTime, Utc};
 use displaydoc::Display;
+use indexmap::IndexMap;
 pub use jwt_simple::{
    algorithms::{HS256Key, MACLike},
    claims::{Claims, JWTClaims},
@ -146,7 +146,7 @@ pub struct AtticAccess {
    /// Cache permissions.
    ///
    /// Keys here may include wildcards.
-    caches: HashMap<CacheNamePattern, CachePermission>,
+    caches: IndexMap<CacheNamePattern, CachePermission>,
 }

 /// Permission to a single cache.
@ -274,7 +274,7 @@ impl Token {
        &mut self,
        pattern: CacheNamePattern,
    ) -> &mut CachePermission {
-        use std::collections::hash_map::Entry;
+        use indexmap::map::Entry;

        let access = self.attic_access_mut();
        match access.caches.entry(pattern) {
--- a/token/src/tests.rs
+++ b/token/src/tests.rs
@ -21,6 +21,8 @@ fn test_basic() {
        "exp": 4102324986,
        "https://jwt.attic.rs/v1": {
          "caches": {
+            "all-*": {"r":1},
+            "all-ci-*": {"w":1},
            "cache-rw": {"r":1,"w":1},
            "cache-ro": {"r":1},
            "team-*": {"r":1,"w":1,"cc":1}
@ -29,7 +31,29 @@ fn test_basic() {
      }
    */

-    let token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtZW93IiwiZXhwIjo0MTAyMzI0OTg2LCJodHRwczovL2p3dC5hdHRpYy5ycy92MSI6eyJjYWNoZXMiOnsiY2FjaGUtcnciOnsiciI6MSwidyI6MX0sImNhY2hlLXJvIjp7InIiOjF9LCJ0ZWFtLSoiOnsiciI6MSwidyI6MSwiY2MiOjF9fX19.UlsIM9bQHr9SXGAcSQcoVPo9No8Zhh6Y5xfX8vCmKmA";
+    let token = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjQxMDIzMjQ5ODYsImh0dHBzOi8vand0LmF0dGljLnJzL3YxIjp7ImNhY2hlcyI6eyJhbGwtKiI6eyJyIjoxfSwiYWxsLWNpLSoiOnsidyI6MX0sImNhY2hlLXJvIjp7InIiOjF9LCJjYWNoZS1ydyI6eyJyIjoxLCJ3IjoxfSwidGVhbS0qIjp7ImNjIjoxLCJyIjoxLCJ3IjoxfX19LCJpYXQiOjE3MTY2NjA1ODksInN1YiI6Im1lb3cifQ.8vtxp_1OEYdcnkGPM4c9ORXooJZV7DOTS4NRkMKN8mw";
+
+    // NOTE(cole-h): check that we get a consistent iteration order when getting permissions for
+    // caches -- this depends on the order of the fields in the token, but should otherwise be
+    // consistent between iterations
+    let mut was_ever_wrong = false;
+    for _ in 0..=1_000 {
+        // NOTE(cole-h): we construct a new Token every iteration in order to get different "random
+        // state"
+        let decoded = Token::from_jwt(token, &dec_key).unwrap();
+        let perm_all_ci = decoded.get_permission_for_cache(&cache! { "all-ci-abc" });
+
+        // NOTE(cole-h): if the iteration order of the token is inconsistent, the permissions may be
+        // retrieved from the `all-ci-*` pattern (which only allows writing/pushing), even though
+        // the `all-*` pattern (which only allows reading/pulling) is specified first
+        if perm_all_ci.require_pull().is_err() || perm_all_ci.require_push().is_ok() {
+            was_ever_wrong = true;
+        }
+    }
+    assert!(
+        !was_ever_wrong,
+        "Iteration order should be consistent to prevent random auth failures (and successes)"
+    );

    let decoded = Token::from_jwt(token, &dec_key).unwrap();
Author	SHA1	Message	Date
Nikodem Rabuliński	f2cfa79c61	Migrate to lix	2024-07-19 21:20:46 +02:00
Zhaofeng Li	6139576a3c	Merge pull request #146 from zhaofengli/install-ci-hotfix ci-installer: Remove drvPath from fake derivation	2024-07-09 12:27:54 -04:00
Zhaofeng Li	4168282329	.github: Use latest macOS runners	2024-07-09 12:26:18 -04:00
Zhaofeng Li	ee8f374737	Merge pull request #139 from NLincoln/sqlite-speedup sqlite-specific performance tuning	2024-07-09 08:12:19 -04:00
Zhaofeng Li	71396b9518	ci-installer: Remove drvPath from fake derivation Also hotfixes the current install-attic-ci.sh. Fixes #145. Ref: `f923ed6b6a`	2024-07-09 07:56:03 -04:00
A cursed quail	759dbc9f7e	sqlite-specific performance tuning	2024-06-08 22:05:37 -05:00
Zhaofeng Li	717cc95983	Merge pull request #138 from zhaofengli/axum-0.7 Various bumps	2024-06-01 16:04:00 -06:00
Zhaofeng Li	14cb5f9e46	Trivial semver-incompatible upgrades	2024-06-01 13:47:27 -06:00
Zhaofeng Li	9a6b2cbf1d	server: Upgrade aws-sdk-rust	2024-06-01 13:47:27 -06:00
Zhaofeng Li	bc22e00a3b	server: Upgrade to Axum 0.7	2024-06-01 13:47:27 -06:00
Zhaofeng Li	cb1b80e989	attic/Cargo.toml: Activate tokio/rt with nix-store Makes `cargo check` inside the crate work.	2024-06-01 13:47:27 -06:00
Zhaofeng Li	2a6b9c592b	.editorconfig: Fix indentation	2024-06-01 13:47:27 -06:00
Zhaofeng Li	4ffeb50f50	Cargo.lock: Update	2024-06-01 13:47:27 -06:00
Zhaofeng Li	a1a521c32f	flake.lock: Update crane	2024-06-01 13:47:27 -06:00
Zhaofeng Li	ff3ce2c0b8	attic/build.rs: Fix filename in workaround comment	2024-06-01 13:47:27 -06:00
Zhaofeng Li	6603ee14ed	Merge pull request #134 from srhb/trim-token-file trim token-file contents	2024-06-01 08:51:22 -06:00
Zhaofeng Li	ec0469cad0	Merge pull request #135 from cole-h/token-use-indexmap token: switch to using IndexMap for consistent ordering of entries	2024-06-01 08:24:31 -06:00
Zhaofeng Li	0558269391	Merge pull request #137 from cole-h/log-stream-errors Log stream errors	2024-06-01 08:17:54 -06:00
Zhaofeng Li	3907b31157	Merge pull request #116 from ixmatus/parnell/fix-gc-bug gc.rs: `LIMIT` number of `orphan_chunks`, fixes #115	2024-06-01 08:13:20 -06:00
Cole Helbling	a4f2cae9dd	Log stream errors	2024-05-28 11:32:43 -07:00
Cole Helbling	18dedcc30b	token: switch to using IndexMap for consistent ordering of entries Because of the random ordering of HashMap, if you have overlapping token permissions, it is possible to randomly pick one that leads to some operation working intermittently (see https://github.com/zhaofengli/attic/issues/133 for an example of this). By using an IndexMap instead, we make "iteration order of the key-value pairs [...] independent of the hash values of the keys" (from the indexmap crate docs [1]), which leads to more predictable behavior. [1]: https://docs.rs/indexmap/latest/indexmap/	2024-05-25 11:26:40 -07:00
Cole Helbling	f18f581188	token: test that permissions iteration order is consistent	2024-05-25 11:26:40 -07:00
Sarah Brofeldt	a2a2011b5f	trim token-file contents Any ordinary file will contain at least a trailing newline which cannot be included in the bearer header. Trim all leading/trailing whitespace.	2024-05-15 08:17:59 +02:00
Parnell Springmeyer	d3ffcf885c	Choose a more sensible limit for MySQL, suggested by @baloo	2024-02-28 14:06:31 -08:00
Parnell Springmeyer	2705d1d90b	Limit the chunk finding query, not the GC mark query	2024-02-28 12:53:27 -08:00
Parnell Springmeyer	47f17e0900	gc.rs: LIMIT number of `orphan_chunk_ids`; fixes #115	2024-02-28 10:37:36 -08:00