Fix wuffs external's wrapped-archive bug; install Flatpak runtimes in CI
GitHub's archive-by-commit endpoint wraps content in an extra "<repo>-<commit>/" directory, which skia's ninja build doesn't expect -- confirmed by a real sandboxed build reaching skia's actual compile step (package 65/69) and failing with a confusing "missing file" error, not a hash mismatch. Switched wuffs-mirror-release-c to the same git+tar mechanism already used for the three googlesource externals (libyuv, piex, angle's zlib fork), which produces the correct unwrapped layout regardless of the reproducibility issue that originally forced git+tar for those three. Also fixes the Dagger module's second real CI failure: a fresh container has no Flatpak runtimes installed at all. Build() now adds the flathub remote and installs org.freedesktop.Platform/Sdk//26.08 and the rust-stable extension first (mirroring `task install-tools`), cached across Dagger runs via a cache volume at ~/.local/share/flatpak. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
63b3bc0df5
commit
cc5a0014c4
5 changed files with 60 additions and 48 deletions
|
|
@ -46,6 +46,26 @@ func (m *FlathubLadybird) Build(ctx context.Context,
|
|||
bundleName := appID + ".flatpak"
|
||||
|
||||
return nixContainer(src).
|
||||
// A fresh container has no Flatpak runtimes at all -- mirrors what
|
||||
// `task install-tools` does on a dev machine. Cached across Dagger
|
||||
// runs (org.freedesktop.Platform/Sdk alone are >1GB) since the
|
||||
// runtime version rarely changes.
|
||||
WithMountedCache("/root/.local/share/flatpak", dag.CacheVolume("flathub-ladybird-flatpak-runtimes")).
|
||||
WithExec([]string{
|
||||
"nix", "develop", "--command",
|
||||
"flatpak", "remote-add", "--user", "--if-not-exists", "flathub",
|
||||
"https://flathub.org/repo/flathub.flatpakrepo",
|
||||
}).
|
||||
WithExec([]string{
|
||||
"nix", "develop", "--command",
|
||||
"flatpak", "install", "-y", "--user", "flathub",
|
||||
"org.freedesktop.Platform//26.08", "org.freedesktop.Sdk//26.08",
|
||||
}).
|
||||
WithExec([]string{
|
||||
"nix", "develop", "--command",
|
||||
"flatpak", "install", "-y", "--user", "flathub",
|
||||
"org.freedesktop.Sdk.Extension.rust-stable//26.08",
|
||||
}).
|
||||
WithMountedCache("/repo/Build/caches/vcpkg-binary-cache", dag.CacheVolume("flathub-ladybird-vcpkg-binary-cache")).
|
||||
WithMountedCache("/root/.cache/ccache", dag.CacheVolume("flathub-ladybird-ccache")).
|
||||
WithExec([]string{
|
||||
|
|
|
|||
|
|
@ -108,15 +108,23 @@
|
|||
"dest": "vcpkg-git-externals/angle-zlib",
|
||||
"x-comment": "Same situation as libyuv/piex above, but for the angle port (chromium's zlib fork, one of angle's own third_party deps) rather than skia."
|
||||
},
|
||||
{
|
||||
"type": "git",
|
||||
"url": "https://github.com/google/wuffs-mirror-release-c",
|
||||
"commit": "e3f919ccfe3ef542cfc983a82146070258fb57f8",
|
||||
"dest": "vcpkg-git-externals/wuffs",
|
||||
"x-comment": "Skia's third googlesource/git-fetched external, but hosted on GitHub -- which DOES serve a stable archive-by-commit endpoint, so this one was originally a plain 'file' source pointed at it. That broke the real flatpak-builder sandboxed build anyway: GitHub's archive wraps content in an extra 'wuffs-mirror-release-c-<commit>/' directory that skia's ninja build doesn't expect, so files end up one level too deep and the build fails with a confusing 'missing file' error rather than a hash mismatch (confirmed by a real sandboxed build run). git+tar (matching libyuv/piex/angle-zlib) produces the correct unwrapped layout, so it's used here too even though the reproducibility problem that forced git+tar for the other three doesn't apply to this one."
|
||||
},
|
||||
{
|
||||
"type": "shell",
|
||||
"commands": [
|
||||
"mkdir -p vcpkg/downloads",
|
||||
"tar czf vcpkg/downloads/libyuv-d98915a654d3564e4802a0004add46221c4e4348.tar.gz -C vcpkg-git-externals/libyuv --exclude=.git .",
|
||||
"tar czf vcpkg/downloads/skia-bb217acdca1cc0c16b704669dd6f91a1b509c406.tar.gz -C vcpkg-git-externals/piex --exclude=.git .",
|
||||
"tar czf vcpkg/downloads/angle-4028ebf8710ee39d2286cb0f847f9b95c59f84d8.tar.gz -C vcpkg-git-externals/angle-zlib --exclude=.git ."
|
||||
"tar czf vcpkg/downloads/angle-4028ebf8710ee39d2286cb0f847f9b95c59f84d8.tar.gz -C vcpkg-git-externals/angle-zlib --exclude=.git .",
|
||||
"tar czf vcpkg/downloads/skia-e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz -C vcpkg-git-externals/wuffs --exclude=.git ."
|
||||
],
|
||||
"x-comment": "Synthesizes the exact tarballs vcpkg's skia portfile expects to find already cached in downloads/, from the hash-pinned git checkouts above -- no network involved, so this is safe to run in the sandboxed build. Verified: a full (non-download-only) `vcpkg install` against these exact sources, outside the flatpak sandbox but with the sandbox's own VCPKG_ROOT/downloads/ layout, completed successfully for all 69 ports including skia, using tarballs produced this same way."
|
||||
"x-comment": "Synthesizes the exact tarballs vcpkg's portfiles expect to find already cached in downloads/, from the hash-pinned git checkouts above -- no network involved, so this is safe to run in the sandboxed build."
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
|
|
@ -128,7 +136,7 @@
|
|||
"@@CARGO_SOURCES@@",
|
||||
"@@VCPKG_SOURCES@@"
|
||||
],
|
||||
"x-comment": "Offline vendoring for vcpkg is believed complete: a full local `vcpkg install` (not just --only-downloads) against the exact sources/config in this module succeeded for all 69 ports, including compiling skia, with the only build-time network need (angle's `pip install ply`) also vendored above. NOT YET verified end-to-end inside the actual flatpak-builder sandbox itself (only outside it, in a distrobox with real network, using the same VCPKG_ROOT/downloads layout) -- that's the next thing to confirm. Rust deps ARE vendored and manifest-verified (see sources/cargo-sources.json). See README.md for full details and what's still open (notably: -DENABLE_QT=ON doesn't appear to pull Qt into the resolved package set)."
|
||||
"x-comment": "Offline vendoring for vcpkg: a real flatpak-builder sandboxed build reached all the way to skia's actual ninja compile step (package 65/69) using these sources before finding the wuffs wrapping-directory bug described above; that's now fixed the same way as the other three git externals. Not yet re-confirmed end to end after that fix -- that's the immediate next thing to verify. Rust deps ARE vendored and manifest-verified (see sources/cargo-sources.json). See README.md for full details and what's still open (notably: -DENABLE_QT=ON doesn't appear to pull Qt into the resolved package set)."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -112,15 +112,23 @@
|
|||
"dest": "vcpkg-git-externals/angle-zlib",
|
||||
"x-comment": "Same situation as libyuv/piex above, but for the angle port (chromium's zlib fork, one of angle's own third_party deps) rather than skia."
|
||||
},
|
||||
{
|
||||
"type": "git",
|
||||
"url": "https://github.com/google/wuffs-mirror-release-c",
|
||||
"commit": "e3f919ccfe3ef542cfc983a82146070258fb57f8",
|
||||
"dest": "vcpkg-git-externals/wuffs",
|
||||
"x-comment": "Skia's third googlesource/git-fetched external, but hosted on GitHub -- which DOES serve a stable archive-by-commit endpoint, so this one was originally a plain 'file' source pointed at it. That broke the real flatpak-builder sandboxed build anyway: GitHub's archive wraps content in an extra 'wuffs-mirror-release-c-<commit>/' directory that skia's ninja build doesn't expect, so files end up one level too deep and the build fails with a confusing 'missing file' error rather than a hash mismatch (confirmed by a real sandboxed build run). git+tar (matching libyuv/piex/angle-zlib) produces the correct unwrapped layout, so it's used here too even though the reproducibility problem that forced git+tar for the other three doesn't apply to this one."
|
||||
},
|
||||
{
|
||||
"type": "shell",
|
||||
"commands": [
|
||||
"mkdir -p vcpkg/downloads",
|
||||
"tar czf vcpkg/downloads/libyuv-d98915a654d3564e4802a0004add46221c4e4348.tar.gz -C vcpkg-git-externals/libyuv --exclude=.git .",
|
||||
"tar czf vcpkg/downloads/skia-bb217acdca1cc0c16b704669dd6f91a1b509c406.tar.gz -C vcpkg-git-externals/piex --exclude=.git .",
|
||||
"tar czf vcpkg/downloads/angle-4028ebf8710ee39d2286cb0f847f9b95c59f84d8.tar.gz -C vcpkg-git-externals/angle-zlib --exclude=.git ."
|
||||
"tar czf vcpkg/downloads/angle-4028ebf8710ee39d2286cb0f847f9b95c59f84d8.tar.gz -C vcpkg-git-externals/angle-zlib --exclude=.git .",
|
||||
"tar czf vcpkg/downloads/skia-e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz -C vcpkg-git-externals/wuffs --exclude=.git ."
|
||||
],
|
||||
"x-comment": "Synthesizes the exact tarballs vcpkg's skia portfile expects to find already cached in downloads/, from the hash-pinned git checkouts above -- no network involved, so this is safe to run in the sandboxed build. Verified: a full (non-download-only) `vcpkg install` against these exact sources, outside the flatpak sandbox but with the sandbox's own VCPKG_ROOT/downloads/ layout, completed successfully for all 69 ports including skia, using tarballs produced this same way."
|
||||
"x-comment": "Synthesizes the exact tarballs vcpkg's portfiles expect to find already cached in downloads/, from the hash-pinned git checkouts above -- no network involved, so this is safe to run in the sandboxed build."
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
|
|
@ -2825,13 +2833,6 @@
|
|||
"dest": "vcpkg/downloads",
|
||||
"dest-filename": "simdutf-simdutf-v9.1.0.tar.gz"
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"url": "https://github.com/google/wuffs-mirror-release-c/archive/e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz",
|
||||
"sha256": "e849dab1f372f16b782ba7528e7f70281e35425bd2d644c92a36fb95909f98db",
|
||||
"dest": "vcpkg/downloads",
|
||||
"dest-filename": "skia-e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz"
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"url": "https://sqlite.org/2026/sqlite-autoconf-3530300.tar.gz",
|
||||
|
|
@ -2861,7 +2862,7 @@
|
|||
"dest-filename": "webmproject-libwebp-v1.6.0.tar.gz"
|
||||
}
|
||||
],
|
||||
"x-comment": "Offline vendoring for vcpkg is believed complete: a full local `vcpkg install` (not just --only-downloads) against the exact sources/config in this module succeeded for all 69 ports, including compiling skia, with the only build-time network need (angle's `pip install ply`) also vendored above. NOT YET verified end-to-end inside the actual flatpak-builder sandbox itself (only outside it, in a distrobox with real network, using the same VCPKG_ROOT/downloads layout) -- that's the next thing to confirm. Rust deps ARE vendored and manifest-verified (see sources/cargo-sources.json). See README.md for full details and what's still open (notably: -DENABLE_QT=ON doesn't appear to pull Qt into the resolved package set)."
|
||||
"x-comment": "Offline vendoring for vcpkg: a real flatpak-builder sandboxed build reached all the way to skia's actual ninja compile step (package 65/69) using these sources before finding the wuffs wrapping-directory bug described above; that's now fixed the same way as the other three git externals. Not yet re-confirmed end to end after that fix -- that's the immediate next thing to verify. Rust deps ARE vendored and manifest-verified (see sources/cargo-sources.json). See README.md for full details and what's still open (notably: -DENABLE_QT=ON doesn't appear to pull Qt into the resolved package set)."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -440,13 +440,6 @@
|
|||
"dest": "vcpkg/downloads",
|
||||
"dest-filename": "simdutf-simdutf-v9.1.0.tar.gz"
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"url": "https://github.com/google/wuffs-mirror-release-c/archive/e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz",
|
||||
"sha256": "e849dab1f372f16b782ba7528e7f70281e35425bd2d644c92a36fb95909f98db",
|
||||
"dest": "vcpkg/downloads",
|
||||
"dest-filename": "skia-e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz"
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"url": "https://sqlite.org/2026/sqlite-autoconf-3530300.tar.gz",
|
||||
|
|
|
|||
|
|
@ -49,30 +49,23 @@ EXCLUDE_DIR_NAMES = {"tools", "git-tmp", "temp", "__pycache__"}
|
|||
# from anywhere, so there's nothing to vendor; just don't flag them as unmatched.
|
||||
NOT_A_DOWNLOAD = {"parsetab.py"}
|
||||
|
||||
# Kind 3a: ports vcpkg fetches via git clone + packages into downloads/<file> itself,
|
||||
# where the origin also serves a stable/reproducible archive-by-commit endpoint (GitHub
|
||||
# does). filename (as it lands in downloads/) -> (re-fetch url, sha256 of that url's
|
||||
# content). Hand-verified against the pinned vcpkg commit in manifest-template.json;
|
||||
# re-derive if that pin ever moves and this script reports these filenames as unmatched.
|
||||
GIT_DERIVED = {
|
||||
"skia-e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz": (
|
||||
"https://github.com/google/wuffs-mirror-release-c/archive/e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz",
|
||||
"e849dab1f372f16b782ba7528e7f70281e35425bd2d644c92a36fb95909f98db",
|
||||
),
|
||||
}
|
||||
|
||||
# Kind 3b: same as above, but the origin (googlesource/Gitiles) serves a DIFFERENT set
|
||||
# of bytes for its "+archive" endpoint on every single request (confirmed: two fetches
|
||||
# of the identical commit produced two different sha256 hashes), so it can never satisfy
|
||||
# a hash-pinned flatpak "file" source. These are handled directly in
|
||||
# manifest-template.json instead, as a hash-pinned "git" source (commit, not download
|
||||
# bytes, is the integrity guarantee) plus a "shell" source that `tar`s the checkout into
|
||||
# the exact filename vcpkg expects in downloads/ -- skipped here, not reported as
|
||||
# unmatched.
|
||||
# Kind 3: ports vcpkg fetches via git clone + packages into downloads/<file> itself, so
|
||||
# there's no distfile URL to point a hash-pinned flatpak "file" source at even when the
|
||||
# origin's archive-by-commit endpoint is stable (GitHub's is; googlesource's Gitiles is
|
||||
# NOT -- confirmed: two fetches of the identical commit produced two different sha256
|
||||
# hashes -- but GitHub's turned out to have the opposite problem instead: it wraps
|
||||
# content in an extra "<repo>-<commit>/" directory that skia's build doesn't expect,
|
||||
# breaking the ninja build several steps later with a confusing "missing file" error
|
||||
# rather than a hash mismatch). All of these are handled directly in
|
||||
# manifest-template.json instead: a hash-pinned "git" source (commit, not download
|
||||
# bytes, is the integrity guarantee -- and produces the unwrapped layout skia wants) plus
|
||||
# a "shell" source that `tar`s the checkout into the exact filename vcpkg expects in
|
||||
# downloads/ -- skipped here, not reported as unmatched.
|
||||
GIT_SOURCE_HANDLED_ELSEWHERE = {
|
||||
"libyuv-d98915a654d3564e4802a0004add46221c4e4348.tar.gz",
|
||||
"skia-bb217acdca1cc0c16b704669dd6f91a1b509c406.tar.gz",
|
||||
"angle-4028ebf8710ee39d2286cb0f847f9b95c59f84d8.tar.gz",
|
||||
"skia-e3f919ccfe3ef542cfc983a82146070258fb57f8.tar.gz",
|
||||
}
|
||||
|
||||
DOWNLOADING_ARROW_RE = re.compile(r"^Downloading (\S+) -> (\S+)$")
|
||||
|
|
@ -145,14 +138,11 @@ def main() -> None:
|
|||
if entry.name in GIT_SOURCE_HANDLED_ELSEWHERE or entry.name in NOT_A_DOWNLOAD:
|
||||
continue
|
||||
|
||||
if entry.name in GIT_DERIVED:
|
||||
url, sha256 = GIT_DERIVED[entry.name]
|
||||
else:
|
||||
url = TOOL_FILENAME_OVERRIDES.get(entry.name) or tool_urls.get(entry.name) or log_urls.get(entry.name)
|
||||
if url is None:
|
||||
unmatched.append(entry.name)
|
||||
continue
|
||||
sha256 = sha256_of(entry)
|
||||
url = TOOL_FILENAME_OVERRIDES.get(entry.name) or tool_urls.get(entry.name) or log_urls.get(entry.name)
|
||||
if url is None:
|
||||
unmatched.append(entry.name)
|
||||
continue
|
||||
sha256 = sha256_of(entry)
|
||||
|
||||
sources.append({
|
||||
"type": "file",
|
||||
|
|
|
|||
Loading…
Reference in a new issue