From f76c3e17a74ed46af75cc8d0151277a8499c1cc5 Mon Sep 17 00:00:00 2001
From: Nathan McCarty <nathan@mccarty.io>
Date: Sun, 9 Oct 2022 21:43:38 -0400
Subject: [PATCH] Start of GPU passthrough plan

---
 machines/levitation/configuration.nix |  3 +
 modules/linux/default.nix             |  2 +
 modules/linux/hardware.nix            | 36 ++++++++--
 modules/linux/user.nix                | 95 ++++++++++++++-------------
 4 files changed, 85 insertions(+), 51 deletions(-)

diff --git a/machines/levitation/configuration.nix b/machines/levitation/configuration.nix
index 431b421..a0e7bb2 100644
--- a/machines/levitation/configuration.nix
+++ b/machines/levitation/configuration.nix
@@ -33,6 +33,9 @@
         sshKey = config.sops.secrets."borg-ssh-key".path;
       };
     };
+    hardware = {
+      amdPassthrough = true;
+    };
     config = {
       isDesktop = true;
       setupGrub = true;
diff --git a/modules/linux/default.nix b/modules/linux/default.nix
index 8e5a6c5..0218dd2 100644
--- a/modules/linux/default.nix
+++ b/modules/linux/default.nix
@@ -102,6 +102,8 @@ in
         # Logitech hardware support
         # On by default if the system is a desktop
         logitech = mkDefaultOption "logitech" config.nathan.config.isDesktop;
+        # AMD Single gpu passthrough
+        amdPassthrough = mkEnableOption "logitech";
       };
       # Linux specific configuration
       config = {
diff --git a/modules/linux/hardware.nix b/modules/linux/hardware.nix
index d867aca..67b5bda 100644
--- a/modules/linux/hardware.nix
+++ b/modules/linux/hardware.nix
@@ -4,10 +4,34 @@ let
 in
 with lib;
 {
-  config = {
-    hardware.logitech.wireless = mkIf nw.logitech {
-      enable = true;
-      enableGraphical = true;
-    };
-  };
+  config = mkMerge [
+    {
+      hardware.logitech.wireless = mkIf nw.logitech {
+        enable = true;
+        enableGraphical = true;
+      };
+    }
+    (mkIf nw.amdPassthrough {
+      # Turn on IOMMU and the needed drivers
+      boot = {
+        kernelParams = [ "amd_iommu=on" ];
+        kernelModules = [ "kvm-amd" "vifo-pci" ];
+      };
+      # Enable libvirtd
+      virtualisation.libvirtd = {
+        enable = true;
+        onBoot = "ignore";
+        onShutdown = "shutdown";
+        qemu = {
+          ovmf = {
+            enable = true;
+            package = pkgs.OVMFFull;
+            runAsRoot = true;
+          };
+          swtpm.enable = true;
+        };
+      };
+
+    })
+  ];
 }
diff --git a/modules/linux/user.nix b/modules/linux/user.nix
index 366cb5b..c3e9bb2 100644
--- a/modules/linux/user.nix
+++ b/modules/linux/user.nix
@@ -3,50 +3,55 @@ let
   nc = config.nathan.config;
 in
 with lib; {
-  config = {
-    users = {
-      # If we install the user and the system is hardended, then disable mutable users
-      mutableUsers = !(nc.installUser && nc.harden);
-      # Configure our user, if enabled
-      users."${nc.user}" = mkMerge [
-        (mkIf nc.installUser
-          {
-            # Darwin is special
-            home = if pkgs.stdenv.isDarwin then "/Users/nathan" else "/home/nathan";
-            description = "Nathan McCarty";
-            shell = pkgs.fish;
-            # Linux specific configuration next
+  config = mkMerge [
+    {
+      users = {
+        # If we install the user and the system is hardended, then disable mutable users
+        mutableUsers = !(nc.installUser && nc.harden);
+        # Configure our user, if enabled
+        users."${nc.user}" = mkMerge [
+          (mkIf nc.installUser
+            {
+              # Darwin is special
+              home = if pkgs.stdenv.isDarwin then "/Users/nathan" else "/home/nathan";
+              description = "Nathan McCarty";
+              shell = pkgs.fish;
+              # Linux specific configuration next
+            })
+          (mkIf (nc.installUser && pkgs.stdenv.isLinux) {
+            isNormalUser = true;
+            extraGroups = [ "wheel" "networkmanager" "audio" "docker" "libvirtd" "uinput" "adbusers" "plugdev" ];
+            hashedPassword = "$6$ShBAPGwzKZuB7eEv$cbb3erUqtVGFo/Vux9UwT2NkbVG9VGCxJxPiZFYL0DIc3t4GpYxjkM0M7fFnh.6V8MoSKLM/TvOtzdWbYwI58.";
+            openssh.authorizedKeys.keys = [
+              # yubikey ssh key
+              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILRs6zVljIlQEZ8F+aEBqqbpeFJwCw3JdveZ8TQWfkev cardno:000615938515"
+              # Macbook pro key
+              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGBfkO7kq37RQMT8UE8zQt/vP4Ub7kizLw6niToJwAIe nathan@Nathans-MacBook-Pro.local"
+              # Phone key
+              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFR0zpmBCb0iEOeeI6SBwgucddNzccfQ5Zmdgib5iSmF nix-on-droid@localhost"
+              # Tablet key
+              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKltqneJjfdLjOvnWQC2iP7hP7aTYkURPiR8LFjB7z87 nix-on-droid@localhost"
+            ];
           })
-        (mkIf (nc.installUser && pkgs.stdenv.isLinux) {
-          isNormalUser = true;
-          extraGroups = [ "wheel" "networkmanager" "audio" "docker" "libvirtd" "uinput" "adbusers" "plugdev" ];
-          hashedPassword = "$6$ShBAPGwzKZuB7eEv$cbb3erUqtVGFo/Vux9UwT2NkbVG9VGCxJxPiZFYL0DIc3t4GpYxjkM0M7fFnh.6V8MoSKLM/TvOtzdWbYwI58.";
-          openssh.authorizedKeys.keys = [
-            # yubikey ssh key
-            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILRs6zVljIlQEZ8F+aEBqqbpeFJwCw3JdveZ8TQWfkev cardno:000615938515"
-            # Macbook pro key
-            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGBfkO7kq37RQMT8UE8zQt/vP4Ub7kizLw6niToJwAIe nathan@Nathans-MacBook-Pro.local"
-            # Phone key
-            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFR0zpmBCb0iEOeeI6SBwgucddNzccfQ5Zmdgib5iSmF nix-on-droid@localhost"
-            # Tablet key
-            "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKltqneJjfdLjOvnWQC2iP7hP7aTYkURPiR8LFjB7z87 nix-on-droid@localhost"
-          ];
-        })
-      ];
-    };
-    # If we install the user, enable sudo
-    security.sudo.enable = mkDefault nc.installUser;
-    # If we isntall the user, make them trusted
-    nix.settings.trusted-users =
-      if nc.installUser then [
-        "nathan"
-      ] else [ ];
-    # If we setup the user, install the shell as well
-    environment.systemPackages =
-      if nc.installUser then [
-        pkgs.fish
-      ] else [ ];
-    # Configure the timezone
-    time.timeZone = "America/New_York";
-  };
+        ];
+      };
+      # If we install the user, enable sudo
+      security.sudo.enable = mkDefault nc.installUser;
+      # If we isntall the user, make them trusted
+      nix.settings.trusted-users =
+        if nc.installUser then [
+          "nathan"
+        ] else [ ];
+      # If we setup the user, install the shell as well
+      environment.systemPackages =
+        if nc.installUser then [
+          pkgs.fish
+        ] else [ ];
+      # Configure the timezone
+      time.timeZone = "America/New_York";
+    }
+    (mkIf config.nathan.config.hardware.amdPassthrough {
+      users.users."${nc.user}".extraGroups = [ "libvirtd" ];
+    })
+  ];
 }