From 5bf7d0ef20c700e3841bba0afa5826912896e63a Mon Sep 17 00:00:00 2001 From: "Janik H." Date: Fri, 26 Jul 2024 00:52:31 +0200 Subject: [PATCH] init --- .envrc | 1 + .gitignore | 4 ++ LICENSE | 21 ++++++++ README.md | 3 ++ default.nix | 13 +++++ frontend/default.nix | 10 ++++ frontend/index.html | 106 ++++++++++++++++++++++++++++++++++++++++ frontend/options.html | 85 ++++++++++++++++++++++++++++++++ npins/default.nix | 80 ++++++++++++++++++++++++++++++ npins/sources.json | 11 +++++ optionsJSON.nix | 25 ++++++++++ packagesJSON.nix | 18 +++++++ shell.nix | 10 ++++ updateIndex/default.nix | 8 +++ updateIndex/default.py | 57 +++++++++++++++++++++ 15 files changed, 452 insertions(+) create mode 100644 .envrc create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 default.nix create mode 100644 frontend/default.nix create mode 100644 frontend/index.html create mode 100644 frontend/options.html create mode 100644 npins/default.nix create mode 100644 npins/sources.json create mode 100644 optionsJSON.nix create mode 100644 packagesJSON.nix create mode 100644 shell.nix create mode 100644 updateIndex/default.nix create mode 100644 updateIndex/default.py diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..1d953f4 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use nix diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8161d7e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.direnv +result +packages.json +options.json diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8877ffb --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Dracula Theme + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..909fa45 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# search.forkos.org + +todo diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..2acf879 --- /dev/null +++ b/default.nix @@ -0,0 +1,13 @@ +{ + pins ? import ./npins, + pkgs ? import pins.nixpkgs { }, + nixpkgsSource ? pins.nixpkgs, +}: let + lib = pkgs.lib; +in { + frontend = pkgs.callPackage ./frontend {}; + updateIndex = pkgs.callPackage ./updateIndex { }; + packagesJSON = pkgs.callPackage ./packagesJSON.nix { nixpkgs = nixpkgsSource; }; + optionsJSON = pkgs.callPackage ./optionsJSON.nix { nixpkgs = nixpkgsSource; }; + vmTest = "foo"; +} diff --git a/frontend/default.nix b/frontend/default.nix new file mode 100644 index 0000000..6c77f10 --- /dev/null +++ b/frontend/default.nix @@ -0,0 +1,10 @@ +{ lib +, stdenv +}: stdenv.mkDerivation { + pname = "frontend"; + version = "0.1.0"; + + src = ./.; + + # todo +} diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..269d4e1 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,106 @@ + + + + + + + + + +
+ + +
+
+

Platform

+
+
+
+
+
+
+ +
+
+ + + + + + diff --git a/frontend/options.html b/frontend/options.html new file mode 100644 index 0000000..058393f --- /dev/null +++ b/frontend/options.html @@ -0,0 +1,85 @@ + + + + + + + + + +
+ + +
+ + + + +
+
+
+
+ +
+
+ + + + + + diff --git a/npins/default.nix b/npins/default.nix new file mode 100644 index 0000000..5e7d086 --- /dev/null +++ b/npins/default.nix @@ -0,0 +1,80 @@ +# Generated by npins. Do not modify; will be overwritten regularly +let + data = builtins.fromJSON (builtins.readFile ./sources.json); + version = data.version; + + mkSource = + spec: + assert spec ? type; + let + path = + if spec.type == "Git" then + mkGitSource spec + else if spec.type == "GitRelease" then + mkGitSource spec + else if spec.type == "PyPi" then + mkPyPiSource spec + else if spec.type == "Channel" then + mkChannelSource spec + else + builtins.throw "Unknown source type ${spec.type}"; + in + spec // { outPath = path; }; + + mkGitSource = + { + repository, + revision, + url ? null, + hash, + branch ? null, + ... + }: + assert repository ? type; + # At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository + # In the latter case, there we will always be an url to the tarball + if url != null then + (builtins.fetchTarball { + inherit url; + sha256 = hash; # FIXME: check nix version & use SRI hashes + }) + else + assert repository.type == "Git"; + let + urlToName = + url: rev: + let + matched = builtins.match "^.*/([^/]*)(\\.git)?$" repository.url; + + short = builtins.substring 0 7 rev; + + appendShort = if (builtins.match "[a-f0-9]*" rev) != null then "-${short}" else ""; + in + "${if matched == null then "source" else builtins.head matched}${appendShort}"; + name = urlToName repository.url revision; + in + builtins.fetchGit { + url = repository.url; + rev = revision; + inherit name; + # hash = hash; + }; + + mkPyPiSource = + { url, hash, ... }: + builtins.fetchurl { + inherit url; + sha256 = hash; + }; + + mkChannelSource = + { url, hash, ... }: + builtins.fetchTarball { + inherit url; + sha256 = hash; + }; +in +if version == 3 then + builtins.mapAttrs (_: mkSource) data.pins +else + throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`" diff --git a/npins/sources.json b/npins/sources.json new file mode 100644 index 0000000..3a99845 --- /dev/null +++ b/npins/sources.json @@ -0,0 +1,11 @@ +{ + "pins": { + "nixpkgs": { + "type": "Channel", + "name": "nixpkgs-unstable", + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-24.11pre656183.453402b94f39/nixexprs.tar.xz", + "hash": "16lpbplz7f5ximc7j10qimr043xa89lx470ycb150529828y21wm" + } + }, + "version": 3 +} diff --git a/optionsJSON.nix b/optionsJSON.nix new file mode 100644 index 0000000..b9f155b --- /dev/null +++ b/optionsJSON.nix @@ -0,0 +1,25 @@ +{ lib, stdenv, jq, nixosOptionsDoc, nixpkgs }: let + # NB: This file describes the Nixpkgs manual, which happens to use module + # docs infra originally developed for NixOS. + optionsDoc = nixosOptionsDoc { + inherit (lib.evalModules { + modules = import "${nixpkgs}/nixos/modules/module-list.nix" ++ [ { nixpkgs.hostPlatform = "x86_64-linux"; } ]; + class = "nixpkgsConfig"; + }) options; + # documentType = "none"; + # transformOptions = opt: opt; + }; +in stdenv.mkDerivation { + name = "options.json"; + src = nixpkgs; + + buildInputs = [ jq ]; + + buildPhase = '' + jq -e 'to_entries | map(.key as $k | .value + {"id":($k | gsub("[^a-zA-Z0-9-_]"; "-")), "name":$k})' ${optionsDoc.optionsJSON}/share/doc/nixos/options.json > $out + ''; + + dontUnpack = true; + dontBuild = false; + doCheck = false; +} diff --git a/packagesJSON.nix b/packagesJSON.nix new file mode 100644 index 0000000..0f1c5f6 --- /dev/null +++ b/packagesJSON.nix @@ -0,0 +1,18 @@ +{ stdenv +, nixpkgs +, jq +, lix +}: stdenv.mkDerivation { + name = "packages.json"; + src = nixpkgs; + + buildInputs = [ jq lix ]; + + buildPhase = '' + NIX_STATE_DIR=$TMPDIR NIX_PATH= nix-env -f $src -qa --meta --json --show-trace --arg config "import $src/pkgs/top-level/packages-config.nix" | jq -e 'to_entries | map(.key as $k | .value + {"id":($k | gsub("[^a-zA-Z0-9-_]"; "-")), "name":$k})' > $out + ''; + + dontUnpack = true; + dontBuild = false; + doCheck = false; +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..f452262 --- /dev/null +++ b/shell.nix @@ -0,0 +1,10 @@ +let + sources = import ./npins; + pkgs = import sources.nixpkgs { }; +in pkgs.mkShell { + buildInputs = with pkgs; [ + npins + python3 + python3Packages.meilisearch + ]; +} diff --git a/updateIndex/default.nix b/updateIndex/default.nix new file mode 100644 index 0000000..59201e4 --- /dev/null +++ b/updateIndex/default.nix @@ -0,0 +1,8 @@ +{ python3Packages +, writers +}: writers.writePython3Bin "updateIndex" { + flakeIgnore = [ "E501" ]; + libraries = with python3Packages; [ + meilisearch + ]; +} (builtins.readFile ./default.py) diff --git a/updateIndex/default.py b/updateIndex/default.py new file mode 100644 index 0000000..920b1cb --- /dev/null +++ b/updateIndex/default.py @@ -0,0 +1,57 @@ +import meilisearch +import argparse +import json +import time +import os + + +def meiliTaskStatus(task_uid: int, verbose: bool): + taskInfo = client.get_task(task_uid) + while taskInfo.status != "succeeded" and taskInfo.status != "canceled" and taskInfo.status != "failed": + # no need to hammer the api + time.sleep(3) + taskInfo = client.get_task(task_uid) + if verbose: + print(taskInfo) + else: + if taskInfo.status == "succeeded": + print(f"job {taskInfo.type} for {taskInfo.index_uid} ({taskInfo.uid}) succeeded") + elif taskInfo.status == "canceled": + print(f"job {taskInfo.type} for {taskInfo.index_uid} ({taskInfo.uid}) was canceled") + elif taskInfo.status == "failed": + print(f"job {taskInfo.type} for {taskInfo.index_uid} ({taskInfo.uid}) failed") + else: + raise ValueError(f"unexpected job task type for: {taskInfo}") + + +# todo: error and figure out a sensible default via envrc and maybe a local instance with something like devenv +meiliInstance = os.getenv("MEILI_INSTANCE") +if meiliInstance is None: + meiliInstance = "https://meilisearch.aq0.de/" + +parser = argparse.ArgumentParser( + prog='updateIndex', + description='Updates meilisearch indexs, inteded to be be used for search.forkos.org' +) +parser.add_argument('documentsJsonFile') +parser.add_argument('-k', '--kind', choices=['packages', 'options'], default='packages') +parser.add_argument('-r', '--release', choices=['rolling', 'stable'], default='rolling') +parser.add_argument('-v', '--verbose', action='store_true') +args = parser.parse_args() + +client = meilisearch.Client(meiliInstance) +indexName = f"{args.kind}-{args.release}" + +with open(args.documentsJsonFile) as json_file: + documents = json.load(json_file) + +if args.kind == "packages": + documentResponse = client.index(indexName).add_documents(documents) + indexResponse = client.index(indexName).update_filterable_attributes(['meta']) + meiliTaskStatus(documentResponse.task_uid, args.verbose) + meiliTaskStatus(indexResponse.task_uid, args.verbose) +elif args.kind == "options": + documentResponse = client.index(indexName).add_documents(documents) + meiliTaskStatus(documentResponse.task_uid, args.verbose) +else: + raise ValueError("invalid value for type kind")