root/ncdu-zig
1
0
Fork 0
mirror of https://code.blicky.net/yorhel/ncdu.git synced 2026-01-12 17:08:39 -09:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix handling short reads when uncompressed json

Browse source 
Turns out that zstd can consume compressed data without returning any
decompressed data when the input buffer isn't full enough. I just
increased the input buffer as a workaround.
This commit is contained in:
Yorhel 2024-11-13 15:28:07 +01:00
parent 2fcd7f370c
commit 4bd6e3daba
2 changed files with 12 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/json_import.zig
View file

@ -13,9 +13,9 @@ const c = @import("c.zig").c;
const ZstdReader = struct { const ZstdReader = struct {
ctx: ?*c.ZSTD_DStream, ctx: ?*c.ZSTD_DStream,
in: c.ZSTD_inBuffer, in: c.ZSTD_inBuffer,
// Assumption: ZSTD_decompressStream() can always make progress as long as // Assumption: ZSTD_decompressStream() can always output uncompressed data
// 1/8 of this buffer is filled. // as long as half of this buffer is filled.
inbuf: [c.ZSTD_BLOCKSIZE_MAX + 16]u8, // This ZSTD_DStreamInSize() + a little bit extra inbuf: [c.ZSTD_BLOCKSIZE_MAX * 2 + 1024]u8, // This twice ZSTD_DStreamInSize() + a little bit extra
fn create(head: []const u8) *ZstdReader { fn create(head: []const u8) *ZstdReader {
const r = main.allocator.create(ZstdReader) catch unreachable; const r = main.allocator.create(ZstdReader) catch unreachable;
@ -39,11 +39,15 @@ const ZstdReader = struct {
} }
fn read(r: *ZstdReader, f: std.fs.File, out: []u8) !usize { fn read(r: *ZstdReader, f: std.fs.File, out: []u8) !usize {
if (r.in.size - r.in.pos < r.inbuf.len / 8) { while (r.in.size - r.in.pos < r.inbuf.len / 2) {
if (r.in.pos > 0) {
std.mem.copyForwards(u8, &r.inbuf, r.inbuf[r.in.pos..][0..r.in.size - r.in.pos]); std.mem.copyForwards(u8, &r.inbuf, r.inbuf[r.in.pos..][0..r.in.size - r.in.pos]);
r.in.size -= r.in.pos; r.in.size -= r.in.pos;
r.in.pos = 0; r.in.pos = 0;
r.in.size += try f.read(r.inbuf[r.in.size..]); }
const rd = try f.read(r.inbuf[r.in.size..]);
r.in.size += rd;
if (rd == 0) break;
} }
var arg = c.ZSTD_outBuffer{ var arg = c.ZSTD_outBuffer{
.dst = out.ptr, .dst = out.ptr,

1
src/main.zig
View file

@ -461,7 +461,6 @@ fn readImport(path: [:0]const u8) !void {
else try std.fs.cwd().openFileZ(path, .{}); else try std.fs.cwd().openFileZ(path, .{});
errdefer fd.close(); errdefer fd.close();
// TODO: While we're at it, recognize and handle compressed JSON
var buf: [8]u8 = undefined; var buf: [8]u8 = undefined;
try fd.reader().readNoEof(&buf); try fd.reader().readNoEof(&buf);
if (std.mem.eql(u8, &buf, bin_export.SIGNATURE)) { if (std.mem.eql(u8, &buf, bin_export.SIGNATURE)) {