From 024ea1168a40f3981eb99b37d99a4d8a787a4ea0 Mon Sep 17 00:00:00 2001 From: Lukas Wurzinger <lukas@wrz.one> Date: Sat, 4 Jan 2025 23:51:35 +0100 Subject: [PATCH] various --- common/pubkeys.nix | 2 +- common/puter/puter.nix | 16 +-- devenv.nix | 7 ++ flake.nix | 11 +- hosts/server/vessel/audiocomp.nix | 84 -------------- hosts/server/vessel/musicomp.nix | 51 +++++++++ lib.nix | 10 +- {common => modules}/main-user.nix | 0 modules/musicomp.nix | 120 ++++++++++++++++++++ {common => modules}/user-types.nix | 0 disk/disk.nix => packages/disk/default.nix | 0 {disk => packages/disk}/disk.bash | 0 packages/musicomp/README.md | 1 + packages/musicomp/default.nix | 19 ++++ packages/musicomp/pyproject.toml | 19 ++++ packages/musicomp/src/musicomp/__main__.py | 3 + packages/musicomp/src/musicomp/clean.py | 6 + packages/musicomp/src/musicomp/cli.py | 118 +++++++++++++++++++ packages/musicomp/src/musicomp/replace.py | 6 + packages/musicomp/src/musicomp/todo.py | 68 +++++++++++ packages/musicomp/src/musicomp/transcode.py | 34 ++++++ packages/puter/default.nix | 11 ++ {common => packages}/puter/puter.bash | 0 23 files changed, 485 insertions(+), 101 deletions(-) delete mode 100644 hosts/server/vessel/audiocomp.nix create mode 100644 hosts/server/vessel/musicomp.nix rename {common => modules}/main-user.nix (100%) create mode 100644 modules/musicomp.nix rename {common => modules}/user-types.nix (100%) rename disk/disk.nix => packages/disk/default.nix (100%) rename {disk => packages/disk}/disk.bash (100%) create mode 100644 packages/musicomp/README.md create mode 100644 packages/musicomp/default.nix create mode 100644 packages/musicomp/pyproject.toml create mode 100644 packages/musicomp/src/musicomp/__main__.py create mode 100644 packages/musicomp/src/musicomp/clean.py create mode 100644 packages/musicomp/src/musicomp/cli.py create mode 100644 packages/musicomp/src/musicomp/replace.py create mode 100644 packages/musicomp/src/musicomp/todo.py create mode 100644 packages/musicomp/src/musicomp/transcode.py create mode 100644 packages/puter/default.nix rename {common => packages}/puter/puter.bash (100%) diff --git a/common/pubkeys.nix b/common/pubkeys.nix index 5147644..6672a26 100644 --- a/common/pubkeys.nix +++ b/common/pubkeys.nix @@ -13,5 +13,5 @@ ''; }; - config.pubkeys = import (self + /pubkeys.nix); + config.pubkeys = lib.mkForce (import (self + /pubkeys.nix)); } diff --git a/common/puter/puter.nix b/common/puter/puter.nix index b86647c..3991496 100644 --- a/common/puter/puter.nix +++ b/common/puter/puter.nix @@ -1,13 +1,9 @@ -{pkgs, ...}: let - puter = pkgs.writeShellApplication { - name = "puter"; - runtimeInputs = [ - pkgs.nixos-rebuild - ]; - text = builtins.readFile ./puter.bash; - }; -in { +{ + pkgs, + self, + ... +}: { environment.systemPackages = [ - puter + self.packages.${pkgs.system}.puter ]; } diff --git a/devenv.nix b/devenv.nix index 67c93b5..a80aa4f 100644 --- a/devenv.nix +++ b/devenv.nix @@ -1,4 +1,6 @@ { + languages.python.enable = true; + pre-commit.hooks = { # Nix alejandra.enable = true; @@ -10,5 +12,10 @@ # Shell shellcheck.enable = true; + + # Python + pyright.enable = true; + ruff.enable = true; + ruff-format.enable = true; }; } diff --git a/flake.nix b/flake.nix index 97f69ce..ef31959 100644 --- a/flake.nix +++ b/flake.nix @@ -48,7 +48,7 @@ devenv.root = let devenvRootFileContent = builtins.readFile inputs.devenv-root.outPath; in - pkgs.lib.mkIf (devenvRootFileContent != "") devenvRootFileContent; + self.lib.mkIf (devenvRootFileContent != "") devenvRootFileContent; name = "puter"; @@ -61,7 +61,14 @@ ]; }; - packages.disk = pkgs.callPackage ./disk {}; + packages = + self.lib.genAttrs [ + "puter" + "disk" + "musicomp" + ] ( + name: pkgs.callPackage ./packages/${name} {} + ); }; }; } diff --git a/hosts/server/vessel/audiocomp.nix b/hosts/server/vessel/audiocomp.nix deleted file mode 100644 index ffb3c7e..0000000 --- a/hosts/server/vessel/audiocomp.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - self, - lib, - pkgs, - ... -}: let - audiocomp = pkgs.writeShellApplication { - name = "audiocomp"; - runtimeInputs = [ - pkgs.parallel - pkgs.rsync - pkgs.openssh - ]; - text = let - remoteDir = self.nixosConfigurations.abacus.config.services.navidrome.settings.MusicFolder; - enc = pkgs.writeShellApplication { - name = "enc"; - runtimeInputs = [ - pkgs.opusTools - ]; - text = '' - src=$1 - dst=$src - dst=''${dst%.flac}.opus - dst=/srv/compmusic/''${dst#/srv/music/} - - if [[ -f "$dst" ]]; then - exit - fi - - mkdir --parents -- "$(dirname -- "$dst")" - - echo "encoding ''${src@Q} -> ''${dst@Q}" >&2 - exec opusenc --quiet --bitrate 96.000 -- "$src" "$dst" - ''; - }; - clean = pkgs.writeShellApplication { - name = "clean"; - text = '' - del=$1 - chk=$del - chk=''${chk%.opus}.flac - chk=/srv/music/''${chk#/srv/compmusic/} - - if [[ ! -f "$chk" ]]; then - echo "deleting ''${del@Q}" >&2 - rm --force -- "$del" - fi - ''; - }; - in '' - shopt -s globstar nullglob - - find /srv/music -name '*.flac' -print0 | parallel --null -- ${lib.getExe enc} {} - - find /srv/compmusic -name '*.flac' -exec ${clean} {} \; - - echo syncing >&2 - rsync --verbose --verbose --archive --update --delete --mkpath --exclude lost+found \ - --rsh 'ssh -i /etc/ssh/ssh_host_ed25519_key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' \ - -- /srv/compmusic/ root@wrz.one:${remoteDir} - ''; - }; -in { - systemd.services.audiocomp = { - description = "Compress and sync music"; - serviceConfig = { - Type = "oneshot"; - User = "root"; - Group = "root"; - ExecStart = lib.getExe audiocomp; - }; - }; - - systemd.timers.audiocomp = { - description = "Compress and sync music daily"; - wantedBy = ["timers.target"]; - timerConfig = { - OnCalendar = "*-*-* 03:00:00"; - Persistent = true; - Unit = "audiocomp.service"; - }; - }; -} diff --git a/hosts/server/vessel/musicomp.nix b/hosts/server/vessel/musicomp.nix new file mode 100644 index 0000000..31ad505 --- /dev/null +++ b/hosts/server/vessel/musicomp.nix @@ -0,0 +1,51 @@ +{ + self, + lib, + pkgs, + ... +}: { + services.musicomp.jobs.main = { + music = "/srv/music"; + comp = "/srv/compmusic"; + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + }; + inhibitsSleep = true; + post = let + remoteDir = self.nixosConfigurations.abacus.config.services.navidrome.settings.MusicFolder; + rsyncExe = lib.getExe pkgs.rsync; + in '' + ${rsyncExe} \ + --archive \ + --recursive \ + --delete \ + --update \ + --mkpath \ + --verbose --verbose \ + --exclude lost+found \ + --rsh 'ssh -i /etc/ssh/ssh_host_ed25519_key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' \ + /srv/compmusic/ root@wrz.one:${remoteDir} + ''; + }; + + systemd.services.audiocomp = { + description = "Compress and sync music"; + serviceConfig = { + Type = "oneshot"; + User = "root"; + Group = "root"; + ExecStart = lib.getExe audiocomp; + }; + }; + + systemd.timers.audiocomp = { + description = "Compress and sync music daily"; + wantedBy = ["timers.target"]; + timerConfig = { + OnCalendar = "*-*-* 03:00:00"; + Persistent = true; + Unit = "audiocomp.service"; + }; + }; +} diff --git a/lib.nix b/lib.nix index bee8104..5323e84 100644 --- a/lib.nix +++ b/lib.nix @@ -1,13 +1,13 @@ lib: _: { - findModules = dirs: - builtins.concatMap (dir: - lib.pipe dir [ + findModules = paths: + builtins.concatMap (path: + lib.pipe path [ (lib.fileset.fileFilter ( file: file.hasExt "nix" )) lib.fileset.toList ]) - dirs; + paths; formatHostPort = { host, @@ -30,6 +30,7 @@ lib: _: { inputs, extraModules ? _: [], }: let + modulesDir = ./modules; commonDir = ./common; classesDir = ./classes; hostsDir = ./hosts; @@ -47,6 +48,7 @@ lib: _: { modules = (lib.findModules [ + modulesDir commonDir ./classes/${class} (classesDir + /${class}) diff --git a/common/main-user.nix b/modules/main-user.nix similarity index 100% rename from common/main-user.nix rename to modules/main-user.nix diff --git a/modules/musicomp.nix b/modules/musicomp.nix new file mode 100644 index 0000000..6dc126b --- /dev/null +++ b/modules/musicomp.nix @@ -0,0 +1,120 @@ +{ + self, + lib, + pkgs, + utils, + ... +}: let + inherit (lib) types; + inherit (utils.systemdUtils.unitOptions) unitOption; +in { + options.services.musicomp.jobs = lib.mkOption { + description = '' + Periodic jobs to run with musicomp. + ''; + # type = types.attrsOf (types.submodule ({name, ...}: { + type = types.attrsOf (types.submodule { + options = { + music = lib.mkOption { + type = types.str; + description = '' + Source directory. + ''; + example = "/srv/music"; + }; + + comp = lib.mkOption { + type = types.str; + description = '' + Destination directory for compressed music. + ''; + example = "/srv/comp"; + }; + + post = lib.mkOption { + type = types.lines; + default = ""; + description = '' + Shell commands that are run after compression has finished. + ''; + }; + + workers = lib.mkOption { + type = lib.types.int; + default = 0; + description = '' + Number of workers. + ''; + }; + + timerConfig = lib.mkOption { + type = lib.types.nullOr (lib.types.attrsOf unitOption); + default = { + OnCalendar = "daily"; + Persistent = true; + }; + description = '' + When to run the job. + ''; + }; + + package = lib.mkPackageOption self.packages.${pkgs.system} "musicomp" {}; + + inhibitsSleep = lib.mkOption { + default = false; + type = lib.types.bool; + example = true; + description = '' + Prevents the system from sleeping while running the job. + ''; + }; + }; + description = '' + Periodic compression jobs to run with musicomp. + ''; + default = {}; + }); + }; + + config = { + systemd.services = + lib.mapAttrs' + ( + name: job: + lib.nameValuePair "musicomp-jobs-${name}" { + restartIfChanged = false; + # TODO + wants = ["network-online.target"]; + after = ["network-online.target"]; + + script = '' + ${lib.optionalString job.inhibitsSleep '' + ${lib.getExe' pkgs.systemd "systemd-inhibit"} \ + --mode block \ + --who musicomp \ + --what sleep \ + --why ${lib.escapeShellArg "Scheduled musicomp ${name}"} + ''} + + ${lib.getExe job.package} \ + ${lib.optionalString (job.workers > 0) "--workers ${job.workers}"} \ + -- ${job.music} ${job.comp} + ''; + + postStop = job.post; + + serviceConfig.Type = "oneshot"; + } + ) + config.services.musicomp.jobs; + + systemd.timers = + lib.mapAttrs' + (name: job: + lib.nameValuePair "musicomp-jobs-${name}" { + wantedBy = ["timers.target"]; + inherit (job) timerConfig; + }) + (lib.filterAttrs (_: job: job.timerConfig != null) config.services.musicomp.jobs); + }; +} diff --git a/common/user-types.nix b/modules/user-types.nix similarity index 100% rename from common/user-types.nix rename to modules/user-types.nix diff --git a/disk/disk.nix b/packages/disk/default.nix similarity index 100% rename from disk/disk.nix rename to packages/disk/default.nix diff --git a/disk/disk.bash b/packages/disk/disk.bash similarity index 100% rename from disk/disk.bash rename to packages/disk/disk.bash diff --git a/packages/musicomp/README.md b/packages/musicomp/README.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/packages/musicomp/README.md @@ -0,0 +1 @@ + diff --git a/packages/musicomp/default.nix b/packages/musicomp/default.nix new file mode 100644 index 0000000..92080d7 --- /dev/null +++ b/packages/musicomp/default.nix @@ -0,0 +1,19 @@ +{ + lib, + python3Packages, + opusTools, +}: +python3Packages.buildPythonApplication { + pname = "musicomp"; + version = "0.1.0"; + src = ./.; + pyproject = true; + doCheck = false; + build-system = [python3Packages.hatchling]; + makeWrapperArgs = [ + "--prefix" + "PATH" + ":" + (lib.makeBinPath [opusTools]) + ]; +} diff --git a/packages/musicomp/pyproject.toml b/packages/musicomp/pyproject.toml new file mode 100644 index 0000000..47628af --- /dev/null +++ b/packages/musicomp/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "musicomp" +version = "0.1.0" +description = "" +authors = [ + {name = "Lukas Wurzinger", email = "lukas@wrz.one"} +] +readme = "README.md" +requires-python = ">=3.12" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project.scripts] +musicomp = "musicomp.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/musicomp"] diff --git a/packages/musicomp/src/musicomp/__main__.py b/packages/musicomp/src/musicomp/__main__.py new file mode 100644 index 0000000..4e28416 --- /dev/null +++ b/packages/musicomp/src/musicomp/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/packages/musicomp/src/musicomp/clean.py b/packages/musicomp/src/musicomp/clean.py new file mode 100644 index 0000000..2e2107d --- /dev/null +++ b/packages/musicomp/src/musicomp/clean.py @@ -0,0 +1,6 @@ +from os import PathLike +from pathlib import Path + + +def clean(dst: str | PathLike[str]) -> None: + Path(dst).unlink(missing_ok=True) diff --git a/packages/musicomp/src/musicomp/cli.py b/packages/musicomp/src/musicomp/cli.py new file mode 100644 index 0000000..03218e0 --- /dev/null +++ b/packages/musicomp/src/musicomp/cli.py @@ -0,0 +1,118 @@ +import sys +from multiprocessing import Pool, cpu_count +from argparse import ArgumentParser, ArgumentTypeError, Namespace +from pathlib import Path +from .todo import Todo + + +args = Namespace() + + +def task(todo: Todo) -> None: + todo.run() + if args.verbose: + print("finished", todo, file=sys.stderr) + + +def main(): + def workers_type_func(value: object) -> int: + try: + value = int(value) # pyright: ignore[reportArgumentType] + if value <= 0: + raise ArgumentTypeError(f"{value} is not a positive integer") + except ValueError: + raise ArgumentTypeError(f"{value} is not an integer") + + return value + + parser = ArgumentParser() + + _ = parser.add_argument( + "-w", + "--workers", + default=cpu_count(), + type=workers_type_func, + help="amount of worker processes", + ) + _ = parser.add_argument( + "-i", + "--interactive", + action="store_true", + help="prompt before running", + ) + _ = parser.add_argument( + "-k", + "--keep", + action="store_true", + help="whether source files should be kept if both directories are the same", + ) + _ = parser.add_argument( + "-r", + "--redo", + action="store_true", + help="whether everything should be re-encoded regardless of whether they have already been transcoded", + ) + _ = parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="verbose output", + ) + + _ = parser.add_argument( + "music", + nargs=1, + type=Path, + help="the source directory", + ) + _ = parser.add_argument( + "comp", + nargs=1, + type=Path, + help="the destination directory for compressed files", + ) + + global args + args = parser.parse_args(sys.argv[1:]) + + assert isinstance(args.workers, int) # pyright: ignore[reportAny] + assert isinstance(args.interactive, bool) # pyright: ignore[reportAny] + assert isinstance(args.keep, bool) # pyright: ignore[reportAny] + assert isinstance(args.redo, bool) # pyright: ignore[reportAny] + assert isinstance(args.verbose, int) # pyright: ignore[reportAny] + assert isinstance(args.music, list) # pyright: ignore[reportAny] + assert len(args.music) == 1 # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] + assert isinstance(args.music[0], Path) # pyright: ignore[reportUnknownMemberType] + assert isinstance(args.comp, list) # pyright: ignore[reportAny] + assert len(args.comp) == 1 # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] + assert isinstance(args.comp[0], Path) # pyright: ignore[reportUnknownMemberType] + + src_dir = args.music[0] # pyright: ignore[reportUnknownMemberType] + dst_dir = args.comp[0] # pyright: ignore[reportUnknownMemberType] + + plan = list( + Todo.plan( + src_dir, + dst_dir, + replace=src_dir.samefile(dst_dir) and not args.keep, + redo=args.redo, + ) + ) + + if len(plan) == 0: + print("Nothing to do", file=sys.stderr) + sys.exit(0) + + if args.verbose >= 1 or args.interactive: + print("Plan:", file=sys.stderr) + for todo in plan: + print(todo, file=sys.stderr) + + if args.interactive: + result = input("Do you want to continue? [Y/n] ") + if result.lower() not in ("", "y", "yes"): + sys.exit(1) + + with Pool(args.workers) as pool: + _ = pool.map(task, plan) diff --git a/packages/musicomp/src/musicomp/replace.py b/packages/musicomp/src/musicomp/replace.py new file mode 100644 index 0000000..097f5f3 --- /dev/null +++ b/packages/musicomp/src/musicomp/replace.py @@ -0,0 +1,6 @@ +from os import PathLike +from pathlib import Path + + +def replace(src: str | PathLike[str]) -> None: + Path(src).unlink(missing_ok=True) diff --git a/packages/musicomp/src/musicomp/todo.py b/packages/musicomp/src/musicomp/todo.py new file mode 100644 index 0000000..b9e2142 --- /dev/null +++ b/packages/musicomp/src/musicomp/todo.py @@ -0,0 +1,68 @@ +from pathlib import Path +from os import PathLike +from enum import StrEnum +from dataclasses import dataclass +from typing import Self, override +from collections.abc import Generator +from .transcode import transcode as todo_transcode +from .clean import clean as todo_clean +from .replace import replace as todo_replace + +SRC_SUFFIX = ".flac" +DST_SUFFIX = ".opus" + + +class TodoAct(StrEnum): + TRANSCODE = "transcode" + CLEAN = "clean" + REPLACE = "replace" + + +@dataclass +class Todo: + act: TodoAct + src: str | PathLike[str] + dst: str | PathLike[str] + + @classmethod + def plan( + cls: type[Self], + src_dir: str | PathLike[str], + dst_dir: str | PathLike[str], + replace: bool = False, + redo: bool = False, + ) -> Generator[Self]: + def list_files(dir: str | PathLike[str], suffix: str) -> list[Path]: + files: list[Path] = [] + for f in Path(dir).rglob("*"): + if f.is_file() and f.suffix == suffix: + files.append(f) + return files + + src_files = list_files(src_dir, SRC_SUFFIX) + dst_files = list_files(dst_dir, DST_SUFFIX) + + for f in src_files: + e = dst_dir / (f.relative_to(src_dir).with_suffix(DST_SUFFIX)) + if redo or e not in dst_files: + yield cls(TodoAct.TRANSCODE, f, e) + if replace: + yield cls(TodoAct.REPLACE, f, e) + + for f in dst_files: + e = src_dir / (f.relative_to(dst_dir).with_suffix(SRC_SUFFIX)) + if e not in src_files: + yield cls(TodoAct.CLEAN, f, e) + + def run(self) -> None: + match self.act: + case TodoAct.TRANSCODE: + todo_transcode(self.src, self.dst) + case TodoAct.CLEAN: + todo_clean(self.dst) + case TodoAct.REPLACE: + todo_replace(self.src) + + @override + def __str__(self) -> str: + return f"{self.act} {self.src} -> {self.dst}" diff --git a/packages/musicomp/src/musicomp/transcode.py b/packages/musicomp/src/musicomp/transcode.py new file mode 100644 index 0000000..7ccd74b --- /dev/null +++ b/packages/musicomp/src/musicomp/transcode.py @@ -0,0 +1,34 @@ +from pathlib import Path +from subprocess import run +from os import PathLike + + +class TranscodingError(Exception): + pass + + +def transcode(src: str | PathLike[str], dst: str | PathLike[str]) -> None: + dst = Path(dst) + + dst.parent.mkdir(parents=True, exist_ok=True) + + if dst.is_file(): + dst.unlink() + + opusenc: tuple[str, ...] = ( + "opusenc", + "--quiet", + "--bitrate", + "96.000", + "--music", + "--vbr", + "--comp", + "10", + "--", + str(src), + str(dst), + ) + + cp = run(opusenc) + if cp.returncode != 0: + raise TranscodingError(f"opusenc exited with code {cp.returncode}") diff --git a/packages/puter/default.nix b/packages/puter/default.nix new file mode 100644 index 0000000..bfd5a2c --- /dev/null +++ b/packages/puter/default.nix @@ -0,0 +1,11 @@ +{ + writeShellApplication, + nixos-rebuild, +}: +writeShellApplication { + name = "puter"; + runtimeInputs = [ + nixos-rebuild + ]; + text = builtins.readFile ./puter.bash; +} diff --git a/common/puter/puter.bash b/packages/puter/puter.bash similarity index 100% rename from common/puter/puter.bash rename to packages/puter/puter.bash