From cc392db95419c5b9d9f8a2b7ed8a6d6867fa7432 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 10 Mar 2023 13:15:49 +0100 Subject: [PATCH 1/3] zstd: Check FSE init values If `br.init(s.br.unread())` fails, it may decode bogus data if previous block returned without reading everything from the bit reader. This is used to feed the huff0 table for literal decoding. Return error correctly. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=56870 Add parsing of OSS reported input. --- flate/fuzz_test.go | 6 ++-- fse/decompress.go | 4 ++- fse/fse_test.go | 2 +- internal/fuzz/helpers.go | 35 +++++++++++++++++++++--- s2/dict_test.go | 6 ++-- s2/fuzz_test.go | 6 ++-- s2/lz4convert_test.go | 4 +-- zip/fuzz_test.go | 4 +-- zstd/blockdec.go | 4 +++ zstd/fuzz_test.go | 41 +++++++++++++++------------- zstd/testdata/decode-regression.zip | Bin 7963 -> 8214 bytes zstd/testdata/fuzz/decode-oss.zip | Bin 0 -> 516 bytes 12 files changed, 74 insertions(+), 38 deletions(-) create mode 100644 zstd/testdata/fuzz/decode-oss.zip diff --git a/flate/fuzz_test.go b/flate/fuzz_test.go index fdd41cfb9..527bad25d 100644 --- a/flate/fuzz_test.go +++ b/flate/fuzz_test.go @@ -26,9 +26,9 @@ func TestMain(m *testing.M) { } func FuzzEncoding(f *testing.F) { - fuzz.AddFromZip(f, "testdata/regression.zip", true, false) - fuzz.AddFromZip(f, "testdata/fuzz/encode-raw-corpus.zip", true, testing.Short()) - fuzz.AddFromZip(f, "testdata/fuzz/FuzzEncoding.zip", false, testing.Short()) + fuzz.AddFromZip(f, "testdata/regression.zip", fuzz.TypeRaw, false) + fuzz.AddFromZip(f, "testdata/fuzz/encode-raw-corpus.zip", fuzz.TypeRaw, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/FuzzEncoding.zip", fuzz.TypeGoFuzz, testing.Short()) startFuzz := *fuzzStartF endFuzz := *fuzzEndF diff --git a/fse/decompress.go b/fse/decompress.go index 926f5f153..cc05d0f7e 100644 --- a/fse/decompress.go +++ b/fse/decompress.go @@ -260,7 +260,9 @@ func (s *Scratch) buildDtable() error { // If the buffer is over-read an error is returned. func (s *Scratch) decompress() error { br := &s.bits - br.init(s.br.unread()) + if err := br.init(s.br.unread()); err != nil { + return err + } var s1, s2 decoder // Initialize and decode first state and symbol. diff --git a/fse/fse_test.go b/fse/fse_test.go index a924874d9..5290a5504 100644 --- a/fse/fse_test.go +++ b/fse/fse_test.go @@ -62,7 +62,7 @@ var decTestfiles = []struct { {name: "crash4", fn: func() ([]byte, error) { return os.ReadFile("../testdata/crash4.bin") }, err: "symbolLen (1) too small"}, {name: "crash5", fn: func() ([]byte, error) { return os.ReadFile("../testdata/crash5.bin") }, err: "symbolLen (1) too small"}, {name: "crash6", fn: func() ([]byte, error) { return os.ReadFile("../testdata/dec-crash6.bin") }, err: "newState (32768) outside table size (32768)"}, - {name: "something", fn: func() ([]byte, error) { return os.ReadFile("../testdata/fse-artifact3.bin") }, err: "output size (1048576) > DecompressLimit (1048576)"}, + {name: "something", fn: func() ([]byte, error) { return os.ReadFile("../testdata/fse-artifact3.bin") }, err: "corrupt stream, did not find end of stream"}, } func TestCompress(t *testing.T) { diff --git a/internal/fuzz/helpers.go b/internal/fuzz/helpers.go index 3efadad39..d2f406a6f 100644 --- a/internal/fuzz/helpers.go +++ b/internal/fuzz/helpers.go @@ -6,6 +6,7 @@ package fuzz import ( "archive/zip" "bytes" + "encoding/binary" "fmt" "go/ast" "go/parser" @@ -16,8 +17,20 @@ import ( "testing" ) +type InputType uint8 + +const ( + // TypeRaw indicates that files are raw bytes. + TypeRaw InputType = iota + // TypeGoFuzz indicates files are from Go Fuzzer. + TypeGoFuzz + // TypeOSSFuzz indicates that files are from OSS fuzzer with size before data. + TypeOSSFuzz +) + // AddFromZip will read the supplied zip and add all as corpus for f. -func AddFromZip(f *testing.F, filename string, raw, short bool) { +// Byte slices only. +func AddFromZip(f *testing.F, filename string, t InputType, short bool) { file, err := os.Open(filename) if err != nil { f.Fatal(err) @@ -44,11 +57,25 @@ func AddFromZip(f *testing.F, filename string, raw, short bool) { f.Fatal(err) } rc.Close() - raw := raw + t := t + if t == TypeOSSFuzz { + t = TypeRaw // Fallback + if len(b) >= 4 { + sz := binary.BigEndian.Uint32(b) + if sz == uint32(len(b))-4 { + f.Add(b[4:]) + continue + } + } + } + if bytes.HasPrefix(b, []byte("go test fuzz")) { - raw = false + t = TypeGoFuzz + } else { + t = TypeRaw } - if raw { + + if t == TypeRaw { f.Add(b) continue } diff --git a/s2/dict_test.go b/s2/dict_test.go index c0bab2d87..6643f6716 100644 --- a/s2/dict_test.go +++ b/s2/dict_test.go @@ -404,9 +404,9 @@ func TestDictSize(t *testing.T) { } func FuzzDictBlocks(f *testing.F) { - fuzz.AddFromZip(f, "testdata/enc_regressions.zip", true, false) - fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", true, testing.Short()) - fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", false, testing.Short()) + fuzz.AddFromZip(f, "testdata/enc_regressions.zip", fuzz.TypeRaw, false) + fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", fuzz.TypeRaw, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", fuzz.TypeGoFuzz, testing.Short()) // Fuzzing tweaks: const ( diff --git a/s2/fuzz_test.go b/s2/fuzz_test.go index 830d3656d..1cc6c853b 100644 --- a/s2/fuzz_test.go +++ b/s2/fuzz_test.go @@ -13,9 +13,9 @@ import ( ) func FuzzEncodingBlocks(f *testing.F) { - fuzz.AddFromZip(f, "testdata/enc_regressions.zip", true, false) - fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", true, testing.Short()) - fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", false, testing.Short()) + fuzz.AddFromZip(f, "testdata/enc_regressions.zip", fuzz.TypeRaw, false) + fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", fuzz.TypeRaw, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", fuzz.TypeGoFuzz, testing.Short()) // Fuzzing tweaks: const ( diff --git a/s2/lz4convert_test.go b/s2/lz4convert_test.go index 82ee5bda5..bdc1c11b2 100644 --- a/s2/lz4convert_test.go +++ b/s2/lz4convert_test.go @@ -352,8 +352,8 @@ func BenchmarkCompressBlockReference(b *testing.B) { } func FuzzLZ4Block(f *testing.F) { - fuzz.AddFromZip(f, "testdata/fuzz/lz4-convert-corpus-raw.zip", true, false) - fuzz.AddFromZip(f, "testdata/fuzz/FuzzLZ4Block.zip", false, false) + fuzz.AddFromZip(f, "testdata/fuzz/lz4-convert-corpus-raw.zip", fuzz.TypeRaw, false) + fuzz.AddFromZip(f, "testdata/fuzz/FuzzLZ4Block.zip", fuzz.TypeGoFuzz, false) // Fuzzing tweaks: const ( // Max input size: diff --git a/zip/fuzz_test.go b/zip/fuzz_test.go index ef7b7af24..3517d55e6 100644 --- a/zip/fuzz_test.go +++ b/zip/fuzz_test.go @@ -30,8 +30,8 @@ func FuzzReader(f *testing.F) { } f.Add(b) } - fuzz.AddFromZip(f, "testdata/FuzzReader-raw.zip", true, testing.Short()) - fuzz.AddFromZip(f, "testdata/FuzzReader-enc.zip", false, testing.Short()) + fuzz.AddFromZip(f, "testdata/FuzzReader-raw.zip", fuzz.TypeRaw, testing.Short()) + fuzz.AddFromZip(f, "testdata/FuzzReader-enc.zip", fuzz.TypeGoFuzz, testing.Short()) f.Fuzz(func(t *testing.T, b []byte) { r, err := NewReader(bytes.NewReader(b), int64(len(b))) diff --git a/zstd/blockdec.go b/zstd/blockdec.go index 2445bb4fe..5f272d87f 100644 --- a/zstd/blockdec.go +++ b/zstd/blockdec.go @@ -9,6 +9,7 @@ import ( "encoding/binary" "errors" "fmt" + "hash/crc32" "io" "os" "path/filepath" @@ -442,6 +443,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err } } var err error + if debugDecoder { + println("huff table input:", len(literals), "CRC:", crc32.ChecksumIEEE(literals)) + } huff, literals, err = huff0.ReadTable(literals, huff) if err != nil { println("reading huffman table:", err) diff --git a/zstd/fuzz_test.go b/zstd/fuzz_test.go index 0c8212cc5..1992ba146 100644 --- a/zstd/fuzz_test.go +++ b/zstd/fuzz_test.go @@ -15,18 +15,10 @@ import ( ) func FuzzDecodeAll(f *testing.F) { - fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", true, testing.Short()) - fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", false, testing.Short()) - decLow, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true)) - if err != nil { - f.Fatal(err) - } - defer decLow.Close() - decHi, err := NewReader(nil, WithDecoderLowmem(false), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true)) - if err != nil { - f.Fatal(err) - } - defer decHi.Close() + fuzz.AddFromZip(f, "testdata/decode-regression.zip", fuzz.TypeRaw, false) + fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", fuzz.TypeRaw, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", fuzz.TypeGoFuzz, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/decode-oss.zip", fuzz.TypeOSSFuzz, false) f.Fuzz(func(t *testing.T, b []byte) { // Just test if we crash... @@ -36,8 +28,19 @@ func FuzzDecodeAll(f *testing.F) { t.Fatal(r) } }() - b1, err1 := decLow.DecodeAll(b, nil) - b2, err2 := decHi.DecodeAll(b, nil) + + decLow, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true)) + if err != nil { + f.Fatal(err) + } + defer decLow.Close() + decHi, err := NewReader(nil, WithDecoderLowmem(false), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true)) + if err != nil { + f.Fatal(err) + } + defer decHi.Close() + b1, err1 := decLow.DecodeAll(b, make([]byte, 0, len(b))) + b2, err2 := decHi.DecodeAll(b, make([]byte, 0, len(b))) if err1 != err2 { t.Log(err1, err2) } @@ -60,8 +63,8 @@ func FuzzDecAllNoBMI2(f *testing.F) { } func FuzzDecoder(f *testing.F) { - fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", true, testing.Short()) - fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", false, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", fuzz.TypeRaw, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", fuzz.TypeGoFuzz, testing.Short()) brLow := newBytesReader(nil) brHi := newBytesReader(nil) @@ -112,9 +115,9 @@ func FuzzNoBMI2Dec(f *testing.F) { } func FuzzEncoding(f *testing.F) { - fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-raw.zip", true, testing.Short()) - fuzz.AddFromZip(f, "testdata/comp-crashers.zip", true, false) - fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-encoded.zip", false, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-raw.zip", fuzz.TypeRaw, testing.Short()) + fuzz.AddFromZip(f, "testdata/comp-crashers.zip", fuzz.TypeRaw, false) + fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-encoded.zip", fuzz.TypeGoFuzz, testing.Short()) // Fuzzing tweaks: const ( // Test a subset of encoders. diff --git a/zstd/testdata/decode-regression.zip b/zstd/testdata/decode-regression.zip index 9ea27c8ded3690f878c0d42350bcd5078668a78c..97216ad0479481f574dd0fa13f742905714d3c21 100644 GIT binary patch delta 284 zcmbPjH_c(gOu2ey77+#}1`dXAu~}ivSF4_=GcYh{0I>@YC+CzFm!uY@l~z^hmZTP! zBqtW9>gHzVW#(p9rKae*0mWTXlO1z%{PLZAJ&knD%ndAzjV%pKOihf9jZI7xGz>M9 z6A}V0-dOcNyisv_W8=xj&b=k8@5QsSvbwQNSh0YKgIOR#+W=_cNqaIo0Sb@CKC`oU|?WpP>=_40e`MeJ^%m! delta 32 hcmbQ{Fxzg!Ou5NQ@_8bxYz#oa2!we+db%8l2LOWy1)Tr@ diff --git a/zstd/testdata/fuzz/decode-oss.zip b/zstd/testdata/fuzz/decode-oss.zip new file mode 100644 index 0000000000000000000000000000000000000000..372a23b067c44d045ff2aeb830eeb1cbfb0d713a GIT binary patch literal 516 zcmWIWW@Zs#U}E54_!gTL#(cHvi8=!Vg9Z@00C93oX>mzvQCewLm2OFDaY=GwajI@^ zW?p7)W>sp6t{YI?B{kVGC&w?}$=B0J*Ua3&!r0i-z{J$V*x1;_L_xz)GdUq4;Np!{ z|HB&;D=VuT+k_Phn3x43v<-mv1;8DW7Mu~b-F+>mF3 Date: Fri, 10 Mar 2023 14:18:57 +0100 Subject: [PATCH 2/3] Don't use file (yet) Fail on error nilness mismatch --- zstd/fuzz_test.go | 5 +++-- zstd/testdata/fuzz/decode-corpus-raw.zip | Bin 4952786 -> 4953274 bytes 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zstd/fuzz_test.go b/zstd/fuzz_test.go index 1992ba146..9f3c2a881 100644 --- a/zstd/fuzz_test.go +++ b/zstd/fuzz_test.go @@ -18,7 +18,6 @@ func FuzzDecodeAll(f *testing.F) { fuzz.AddFromZip(f, "testdata/decode-regression.zip", fuzz.TypeRaw, false) fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", fuzz.TypeRaw, testing.Short()) fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", fuzz.TypeGoFuzz, testing.Short()) - fuzz.AddFromZip(f, "testdata/fuzz/decode-oss.zip", fuzz.TypeOSSFuzz, false) f.Fuzz(func(t *testing.T, b []byte) { // Just test if we crash... @@ -42,7 +41,9 @@ func FuzzDecodeAll(f *testing.F) { b1, err1 := decLow.DecodeAll(b, make([]byte, 0, len(b))) b2, err2 := decHi.DecodeAll(b, make([]byte, 0, len(b))) if err1 != err2 { - t.Log(err1, err2) + if (err1 == nil) != (err2 == nil) { + t.Errorf("err low: %v, hi: %v", err1, err2) + } } if err1 != nil { b1, b2 = b1[:0], b2[:0] diff --git a/zstd/testdata/fuzz/decode-corpus-raw.zip b/zstd/testdata/fuzz/decode-corpus-raw.zip index 8314d3f63bb45643f270b2fedcb6daf25a71666a..190061ab3d74cb6a336c685bce18362a1156e875 100644 GIT binary patch delta 698 zcmbu5y-OQ$9EX3oc;`!et5-m-lLmbkR z585uSuEA2UTAVCGp__vaF1k4Ne-Kw8{e}$Mp(yx+XZa1^=izhuZIAo)eUJMD3piL& z03HGgQG{ZYz=l$kp&S*cL=_&O8g|s677y_Vk5PvwsK-+@K!k)wG@%(SXhj>^;XnsE z(S>fv=)p56aH1D3sBohX8a(i#9|L%fLA=0AyxP|<#k94+&Nx=Y-hP~)n9iQp z;lEo2`Sn;V-y-*E3T1c|)vZ#Es!9%LM=oa4Y}_=>WAn@JldO<_=q9GPYe?QVuTnag2d~C3o&#OPg^)U`~P+nPER7}>iw1shyGRN2Z~=W flr!aB%hH`(x9$}V@j~vVPxOh~i4t~2cNzN&YJk@8 delta 208 zcmWN^NlwCW7(nq-pbUzjR0Re91xG++9z>89ltFI830%B{ldxdJiiCtKbjM8$d*b(! z-{S37Ykj|5>%Vx=@#4deL4Y73!klnQgfpVVIOl>mCYM}sO@bsh+>#>A9SVyKS?+n@ zktd#cAxEA9uM~NsM41XzYSd}aq(z%|I(*Qj$0vR4jdM_oqveQ!7>Y0PEk Date: Fri, 10 Mar 2023 17:38:44 +0100 Subject: [PATCH 3/3] Revert useless file change --- zstd/testdata/decode-regression.zip | Bin 8214 -> 7963 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/zstd/testdata/decode-regression.zip b/zstd/testdata/decode-regression.zip index 97216ad0479481f574dd0fa13f742905714d3c21..9ea27c8ded3690f878c0d42350bcd5078668a78c 100644 GIT binary patch delta 32 hcmbQ{Fxzg!Ou5NQ@_8bxYz#oa2!we+db%8l2LOWy1)Tr@ delta 284 zcmbPjH_c(gOu2ey77+#}1`dXAu~}ivSF4_=GcYh{0I>@YC+CzFm!uY@l~z^hmZTP! zBqtW9>gHzVW#(p9rKae*0mWTXlO1z%{PLZAJ&knD%ndAzjV%pKOihf9jZI7xGz>M9 z6A}V0-dOcNyisv_W8=xj&b=k8@5QsSvbwQNSh0YKgIOR#+W=_cNqaIo0Sb@CKC`oU|?WpP>=_40e`MeJ^%m!