fix(nvidia): order CDI generation before nvidia-persistenced to fix boot race (#565)

* fix(nvidia): order CDI generation before nvidia-persistenced to fix boot race

nvidia-persistenced and nvidia-powerd only start if /dev/nvidia* already
exists, but those device nodes aren't created until CDI generation runs.
Without any ordering between the two, persistenced would sometimes lose
that race and stay stopped for the rest of the boot (ublue-os/ucore#419).

nvidia-container-toolkit-base now ships its own nvidia-cdi-refresh
service, which is better than our old ublue-nvctk-cdi.service: it forces
device creation explicitly, retries on failure, and re-runs on driver
upgrades. Switch to it and add a drop-in ordering it before
nvidia-persistenced and nvidia-powerd, so the race can't happen.

Verified on real NVIDIA hardware across multiple reboots.

* chore: Add missing service to nvidia-install.sh

---------

Co-authored-by: Kyle Gospodnetich <me@kylegospodneti.ch>

Containerfile.in

@@ -20,10 +20,8 @@ ARG KMOD_REPO_ARG
 
 COPY certs /tmp/certs
 
-COPY files/usr/lib/systemd/system/ublue-nvctk-cdi.service /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/ublue-nvctk-cdi.service
-COPY files/usr/lib/systemd/system/ublue-nvctk-cdi.service /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/ublue-nvctk-cdi.service
-COPY files/usr/lib/systemd/system-preset/70-ublue-nvctk-cdi.preset /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/70-ublue-nvctk-cdi.preset
-COPY files/usr/lib/systemd/system-preset/70-ublue-nvctk-cdi.preset /tmp/ublue-os-ucore-nvidia/rpmbuild/SOURCES/70-ublue-nvctk-cdi.preset
+COPY files/usr/lib/systemd/system-preset/70-nvidia-cdi-refresh.preset /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/70-nvidia-cdi-refresh.preset
+COPY files/usr/lib/systemd/system/nvidia-cdi-refresh.service.d/10-ublue-ordering.conf /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/10-ublue-ordering.conf
 COPY files/etc/udev/rules.d/60-nvidia-extra-devices-pm.rules /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/60-nvidia-extra-devices-pm.rules
 
 #ifndef CENTOS

build_files/nvidia/nvidia-install.sh

@@ -103,7 +103,7 @@ dnf5 config-manager setopt fedora-nvidia*.enabled=0 nvidia-container-toolkit.ena
 # Disable staging
 dnf5 -y copr disable ublue-os/staging
 
-systemctl enable ublue-nvctk-cdi.service
+systemctl enable nvidia-cdi-refresh.service nvidia-cdi-refresh.path nvidia-persistenced.service
 semodule --verbose --install /usr/share/selinux/packages/nvidia-container.pp
 
 # we must force driver load to fix black screen on boot for nvidia desktops

build_files/nvidia/ublue-os-nvidia-addons/ublue-os-nvidia-addons.spec

@@ -1,5 +1,5 @@
 Name:           ublue-os-nvidia-addons
-Version:        0.14
+Version:        0.15
 Release:        1%{?dist}
 Summary:        Additional files for nvidia driver support
 
@@ -11,8 +11,8 @@ Supplements:    mokutil policycoreutils
 
 Source0:        nvidia-container-toolkit.repo
 Source1:        nvidia-container.pp
-Source2:        ublue-nvctk-cdi.service
-Source3:        70-ublue-nvctk-cdi.preset
+Source2:        70-nvidia-cdi-refresh.preset
+Source3:        10-ublue-ordering.conf
 Source5:        negativo17-fedora-nvidia.repo
 Source6:        60-nvidia-extra-devices-pm.rules
 Source7:        negativo17-epel-nvidia.repo
@@ -46,12 +46,11 @@ install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nega
 
 install -Dm0644 %{SOURCE0} %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 install -Dm0644 %{SOURCE1} %{buildroot}%{_datadir}/ublue-os/%{_datadir}/selinux/packages/nvidia-container.pp
-install -Dm0644 %{SOURCE2} %{buildroot}%{_datadir}/ublue-os/%{_unitdir}/ublue-nvctk-cdi.service
-install -Dm0644 %{SOURCE3} %{buildroot}%{_presetdir}/70-ublue-nvctk-cdi.preset
+install -Dm0644 %{SOURCE2} %{buildroot}%{_presetdir}/70-nvidia-cdi-refresh.preset
+install -Dm0644 %{SOURCE3} %{buildroot}%{_unitdir}/nvidia-cdi-refresh.service.d/10-ublue-ordering.conf
 sed -i 's@enabled=1@enabled=0@g' %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo     %{buildroot}%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_datadir}/selinux/packages/nvidia-container.pp             %{buildroot}%{_datadir}/selinux/packages/nvidia-container.pp
-install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_unitdir}/ublue-nvctk-cdi.service                          %{buildroot}%{_unitdir}/ublue-nvctk-cdi.service
 
 %files
 %if 0%{?rhel}
@@ -68,13 +67,19 @@ install -Dm0644 %{buildroot}%{_datadir}/ublue-os/%{_unitdir}/ublue-nvctk-cdi.ser
 %endif
 %attr(0644,root,root) %{_datadir}/ublue-os/%{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 %attr(0644,root,root) %{_datadir}/ublue-os/%{_datadir}/selinux/packages/nvidia-container.pp
-%attr(0644,root,root) %{_datadir}/ublue-os/%{_unitdir}/ublue-nvctk-cdi.service
 %attr(0644,root,root) %{_sysconfdir}/yum.repos.d/nvidia-container-toolkit.repo
 %attr(0644,root,root) %{_datadir}/selinux/packages/nvidia-container.pp
-%attr(0644,root,root) %{_unitdir}/ublue-nvctk-cdi.service
-%attr(0644,root,root) %{_presetdir}/70-ublue-nvctk-cdi.preset
+%attr(0644,root,root) %{_presetdir}/70-nvidia-cdi-refresh.preset
+%attr(0644,root,root) %{_unitdir}/nvidia-cdi-refresh.service.d/10-ublue-ordering.conf
 
 %changelog
+* Sat Aug 8 2026 Benjamin Sherman <benjamin@holyarmy.org> - 0.15
+- retire ublue-nvctk-cdi.service in favor of upstream nvidia-cdi-refresh.service
+  (shipped by nvidia-container-toolkit-base >= 1.19.1)
+- add drop-in ordering nvidia-cdi-refresh.service before nvidia-persistenced.service and
+  nvidia-powerd.service, fixing an intermittent boot race (ublue-os/ucore#419) where those
+  units' ConditionPathExistsGlob=/dev/nvidia* could be evaluated before device nodes exist
+
 * Mon Dec 8 2025 Benjamin Sherman <benjamin@holyarmy.org> - 0.14
 - add nvidia-driver version 580 LTS repo support
 - remove sway environment file

files/usr/lib/systemd/system-preset/70-nvidia-cdi-refresh.preset

@@ -0,0 +1,2 @@
+enable nvidia-cdi-refresh.service
+enable nvidia-cdi-refresh.path

files/usr/lib/systemd/system-preset/70-ublue-nvctk-cdi.preset

@@ -1 +0,0 @@
-enable ublue-nvctk-cdi.service

files/usr/lib/systemd/system/nvidia-cdi-refresh.service.d/10-ublue-ordering.conf

@@ -0,0 +1,2 @@
+[Unit]
+Before=nvidia-persistenced.service nvidia-powerd.service

files/usr/lib/systemd/system/ublue-nvctk-cdi.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=ublue nvidia container toolkit CDI auto-generation
-ConditionFileIsExecutable=/usr/bin/nvidia-ctk
-After=local-fs.target
-
-[Service]
-Type=oneshot
-ExecStart=/usr/bin/nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
-
-[Install]
-WantedBy=multi-user.target