From 0c1e919a314436f5ab3ae2b29bca3a8e207730ec Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Wed, 17 Dec 2025 15:26:38 -0500 Subject: [PATCH] Add Reader fuzz tests - Fuzz tests are generated for all readers. - Generated tests are executed with initial seeds as part of regular "make test" targets, but no fuzzing happens with regular runs. - Fuzzing can be done by executing "go test -fuzz ". After this PR is merged I will run fuzzing on my machine and will report/fix bugs found by fuzzer in separate PRs. Contributes to https://github.com/splunk/stef/issues/11 --- examples/ints/internal/ints/common.go | 25 +++++++ .../ints/internal/ints/recordwriter_test.go | 75 ++++++++++++------- examples/jsonl/internal/jsonstef/common.go | 25 +++++++ .../internal/jsonstef/recordwriter_test.go | 75 ++++++++++++------- examples/profile/internal/profile/common.go | 25 +++++++ .../internal/profile/samplewriter_test.go | 75 ++++++++++++------- go/otel/otelstef/common.go | 25 +++++++ go/otel/otelstef/metricswriter_test.go | 75 ++++++++++++------- go/otel/otelstef/spanswriter_test.go | 75 ++++++++++++------- stefc/templates/go/common.go.tmpl | 25 +++++++ stefc/templates/go/writer_test.go.tmpl | 75 ++++++++++++------- 11 files changed, 413 insertions(+), 162 deletions(-) diff --git a/examples/ints/internal/ints/common.go b/examples/ints/internal/ints/common.go index d948aa2e..757f5bd9 100644 --- a/examples/ints/internal/ints/common.go +++ b/examples/ints/internal/ints/common.go @@ -5,6 +5,7 @@ import ( "fmt" "math" + "github.com/splunk/stef/go/pkg" "github.com/splunk/stef/go/pkg/schema" ) @@ -83,3 +84,27 @@ type mutateRandomLimiter struct { objectCount int elemCount int } + +var testWriterOpts []pkg.WriterOptions = []pkg.WriterOptions{ + {}, + {Compression: pkg.CompressionZstd}, + {MaxUncompressedFrameByteSize: 500}, + {MaxTotalDictSize: 500}, + { + Compression: pkg.CompressionZstd, + MaxUncompressedFrameByteSize: 500, + MaxTotalDictSize: 500, + }, + {FrameRestartFlags: pkg.RestartDictionaries}, + {FrameRestartFlags: pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, + { + FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, + Compression: pkg.CompressionZstd, + }, + { + FrameRestartFlags: pkg.RestartCodecs, + MaxUncompressedFrameByteSize: 500, + }, +} diff --git a/examples/ints/internal/ints/recordwriter_test.go b/examples/ints/internal/ints/recordwriter_test.go index 9bffc1d4..d98cf882 100644 --- a/examples/ints/internal/ints/recordwriter_test.go +++ b/examples/ints/internal/ints/recordwriter_test.go @@ -43,30 +43,6 @@ func genRecordRecords(random *rand.Rand, schem *schema.Schema) (records []Record func testRecordWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { retVal = true - opts := []pkg.WriterOptions{ - {}, - {Compression: pkg.CompressionZstd}, - {MaxUncompressedFrameByteSize: 500}, - {MaxTotalDictSize: 500}, - { - Compression: pkg.CompressionZstd, - MaxUncompressedFrameByteSize: 500, - MaxTotalDictSize: 500, - }, - {FrameRestartFlags: pkg.RestartDictionaries}, - {FrameRestartFlags: pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, - { - FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, - Compression: pkg.CompressionZstd, - }, - { - FrameRestartFlags: pkg.RestartCodecs, - MaxUncompressedFrameByteSize: 500, - }, - } - random := rand.New(rand.NewPCG(seed, 0)) // Load the schema from the allSchemaContent variable. @@ -83,7 +59,7 @@ func testRecordWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { } wireSchema := schema.NewWireSchema(schem, "Record") - for _, opt := range opts { + for _, opt := range testWriterOpts { t.Run( "", func(t *testing.T) { succeeded := false @@ -95,10 +71,11 @@ func testRecordWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { }() // Write data according to (possibly modified) schema - opt.Schema = &wireSchema + optCpy := opt + optCpy.Schema = &wireSchema buf := &pkg.MemChunkWriter{} - writer, err := NewRecordWriter(buf, opt) + writer, err := NewRecordWriter(buf, optCpy) require.NoError(t, err, "seed %v", seed) // Generate records pseudo-randomly @@ -163,3 +140,47 @@ func TestRecordWriteRead(t *testing.T) { succeeded = testRecordWriteReadSeed(t, seed) } + +func FuzzRecordReader(f *testing.F) { + f.Add([]byte("")) + + random := rand.New(rand.NewPCG(0, 0)) + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(f, err) + + for _, opt := range testWriterOpts { + for i := 0; i <= 3; i++ { + buf := &pkg.MemChunkWriter{} + writer, err := NewRecordWriter(buf, opt) + require.NoError(f, err) + + recCount := (1 << (2 * i)) - 1 + for range recCount { + limiter := &mutateRandomLimiter{} + writer.Record.mutateRandom(random, schem, limiter) + err = writer.Write() + require.NoError(f, err) + } + + err = writer.Flush() + require.NoError(f, err) + + f.Add(buf.Bytes()) + } + } + + f.Fuzz( + func(t *testing.T, data []byte) { + reader, err := NewRecordReader(bytes.NewBuffer(data)) + if err != nil { + return + } + for { + err = reader.Read(pkg.ReadOptions{}) + if err != nil { + break + } + } + }, + ) +} diff --git a/examples/jsonl/internal/jsonstef/common.go b/examples/jsonl/internal/jsonstef/common.go index 4cfe0556..6244d6b5 100644 --- a/examples/jsonl/internal/jsonstef/common.go +++ b/examples/jsonl/internal/jsonstef/common.go @@ -5,6 +5,7 @@ import ( "fmt" "math" + "github.com/splunk/stef/go/pkg" "github.com/splunk/stef/go/pkg/schema" ) @@ -90,3 +91,27 @@ type mutateRandomLimiter struct { objectCount int elemCount int } + +var testWriterOpts []pkg.WriterOptions = []pkg.WriterOptions{ + {}, + {Compression: pkg.CompressionZstd}, + {MaxUncompressedFrameByteSize: 500}, + {MaxTotalDictSize: 500}, + { + Compression: pkg.CompressionZstd, + MaxUncompressedFrameByteSize: 500, + MaxTotalDictSize: 500, + }, + {FrameRestartFlags: pkg.RestartDictionaries}, + {FrameRestartFlags: pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, + { + FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, + Compression: pkg.CompressionZstd, + }, + { + FrameRestartFlags: pkg.RestartCodecs, + MaxUncompressedFrameByteSize: 500, + }, +} diff --git a/examples/jsonl/internal/jsonstef/recordwriter_test.go b/examples/jsonl/internal/jsonstef/recordwriter_test.go index d022764e..3a8055b8 100644 --- a/examples/jsonl/internal/jsonstef/recordwriter_test.go +++ b/examples/jsonl/internal/jsonstef/recordwriter_test.go @@ -43,30 +43,6 @@ func genRecordRecords(random *rand.Rand, schem *schema.Schema) (records []Record func testRecordWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { retVal = true - opts := []pkg.WriterOptions{ - {}, - {Compression: pkg.CompressionZstd}, - {MaxUncompressedFrameByteSize: 500}, - {MaxTotalDictSize: 500}, - { - Compression: pkg.CompressionZstd, - MaxUncompressedFrameByteSize: 500, - MaxTotalDictSize: 500, - }, - {FrameRestartFlags: pkg.RestartDictionaries}, - {FrameRestartFlags: pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, - { - FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, - Compression: pkg.CompressionZstd, - }, - { - FrameRestartFlags: pkg.RestartCodecs, - MaxUncompressedFrameByteSize: 500, - }, - } - random := rand.New(rand.NewPCG(seed, 0)) // Load the schema from the allSchemaContent variable. @@ -83,7 +59,7 @@ func testRecordWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { } wireSchema := schema.NewWireSchema(schem, "Record") - for _, opt := range opts { + for _, opt := range testWriterOpts { t.Run( "", func(t *testing.T) { succeeded := false @@ -95,10 +71,11 @@ func testRecordWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { }() // Write data according to (possibly modified) schema - opt.Schema = &wireSchema + optCpy := opt + optCpy.Schema = &wireSchema buf := &pkg.MemChunkWriter{} - writer, err := NewRecordWriter(buf, opt) + writer, err := NewRecordWriter(buf, optCpy) require.NoError(t, err, "seed %v", seed) // Generate records pseudo-randomly @@ -163,3 +140,47 @@ func TestRecordWriteRead(t *testing.T) { succeeded = testRecordWriteReadSeed(t, seed) } + +func FuzzRecordReader(f *testing.F) { + f.Add([]byte("")) + + random := rand.New(rand.NewPCG(0, 0)) + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(f, err) + + for _, opt := range testWriterOpts { + for i := 0; i <= 3; i++ { + buf := &pkg.MemChunkWriter{} + writer, err := NewRecordWriter(buf, opt) + require.NoError(f, err) + + recCount := (1 << (2 * i)) - 1 + for range recCount { + limiter := &mutateRandomLimiter{} + writer.Record.mutateRandom(random, schem, limiter) + err = writer.Write() + require.NoError(f, err) + } + + err = writer.Flush() + require.NoError(f, err) + + f.Add(buf.Bytes()) + } + } + + f.Fuzz( + func(t *testing.T, data []byte) { + reader, err := NewRecordReader(bytes.NewBuffer(data)) + if err != nil { + return + } + for { + err = reader.Read(pkg.ReadOptions{}) + if err != nil { + break + } + } + }, + ) +} diff --git a/examples/profile/internal/profile/common.go b/examples/profile/internal/profile/common.go index 3e8b60b2..d8440dc7 100644 --- a/examples/profile/internal/profile/common.go +++ b/examples/profile/internal/profile/common.go @@ -5,6 +5,7 @@ import ( "fmt" "math" + "github.com/splunk/stef/go/pkg" "github.com/splunk/stef/go/pkg/schema" ) @@ -146,3 +147,27 @@ type mutateRandomLimiter struct { objectCount int elemCount int } + +var testWriterOpts []pkg.WriterOptions = []pkg.WriterOptions{ + {}, + {Compression: pkg.CompressionZstd}, + {MaxUncompressedFrameByteSize: 500}, + {MaxTotalDictSize: 500}, + { + Compression: pkg.CompressionZstd, + MaxUncompressedFrameByteSize: 500, + MaxTotalDictSize: 500, + }, + {FrameRestartFlags: pkg.RestartDictionaries}, + {FrameRestartFlags: pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, + { + FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, + Compression: pkg.CompressionZstd, + }, + { + FrameRestartFlags: pkg.RestartCodecs, + MaxUncompressedFrameByteSize: 500, + }, +} diff --git a/examples/profile/internal/profile/samplewriter_test.go b/examples/profile/internal/profile/samplewriter_test.go index 3fc3cf8c..f0a32de0 100644 --- a/examples/profile/internal/profile/samplewriter_test.go +++ b/examples/profile/internal/profile/samplewriter_test.go @@ -43,30 +43,6 @@ func genSampleRecords(random *rand.Rand, schem *schema.Schema) (records []Sample func testSampleWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { retVal = true - opts := []pkg.WriterOptions{ - {}, - {Compression: pkg.CompressionZstd}, - {MaxUncompressedFrameByteSize: 500}, - {MaxTotalDictSize: 500}, - { - Compression: pkg.CompressionZstd, - MaxUncompressedFrameByteSize: 500, - MaxTotalDictSize: 500, - }, - {FrameRestartFlags: pkg.RestartDictionaries}, - {FrameRestartFlags: pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, - { - FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, - Compression: pkg.CompressionZstd, - }, - { - FrameRestartFlags: pkg.RestartCodecs, - MaxUncompressedFrameByteSize: 500, - }, - } - random := rand.New(rand.NewPCG(seed, 0)) // Load the schema from the allSchemaContent variable. @@ -83,7 +59,7 @@ func testSampleWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { } wireSchema := schema.NewWireSchema(schem, "Sample") - for _, opt := range opts { + for _, opt := range testWriterOpts { t.Run( "", func(t *testing.T) { succeeded := false @@ -95,10 +71,11 @@ func testSampleWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { }() // Write data according to (possibly modified) schema - opt.Schema = &wireSchema + optCpy := opt + optCpy.Schema = &wireSchema buf := &pkg.MemChunkWriter{} - writer, err := NewSampleWriter(buf, opt) + writer, err := NewSampleWriter(buf, optCpy) require.NoError(t, err, "seed %v", seed) // Generate records pseudo-randomly @@ -163,3 +140,47 @@ func TestSampleWriteRead(t *testing.T) { succeeded = testSampleWriteReadSeed(t, seed) } + +func FuzzSampleReader(f *testing.F) { + f.Add([]byte("")) + + random := rand.New(rand.NewPCG(0, 0)) + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(f, err) + + for _, opt := range testWriterOpts { + for i := 0; i <= 3; i++ { + buf := &pkg.MemChunkWriter{} + writer, err := NewSampleWriter(buf, opt) + require.NoError(f, err) + + recCount := (1 << (2 * i)) - 1 + for range recCount { + limiter := &mutateRandomLimiter{} + writer.Record.mutateRandom(random, schem, limiter) + err = writer.Write() + require.NoError(f, err) + } + + err = writer.Flush() + require.NoError(f, err) + + f.Add(buf.Bytes()) + } + } + + f.Fuzz( + func(t *testing.T, data []byte) { + reader, err := NewSampleReader(bytes.NewBuffer(data)) + if err != nil { + return + } + for { + err = reader.Read(pkg.ReadOptions{}) + if err != nil { + break + } + } + }, + ) +} diff --git a/go/otel/otelstef/common.go b/go/otel/otelstef/common.go index 13e38fb8..53fd42bb 100644 --- a/go/otel/otelstef/common.go +++ b/go/otel/otelstef/common.go @@ -5,6 +5,7 @@ import ( "fmt" "math" + "github.com/splunk/stef/go/pkg" "github.com/splunk/stef/go/pkg/schema" ) @@ -216,3 +217,27 @@ type mutateRandomLimiter struct { objectCount int elemCount int } + +var testWriterOpts []pkg.WriterOptions = []pkg.WriterOptions{ + {}, + {Compression: pkg.CompressionZstd}, + {MaxUncompressedFrameByteSize: 500}, + {MaxTotalDictSize: 500}, + { + Compression: pkg.CompressionZstd, + MaxUncompressedFrameByteSize: 500, + MaxTotalDictSize: 500, + }, + {FrameRestartFlags: pkg.RestartDictionaries}, + {FrameRestartFlags: pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, + { + FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, + Compression: pkg.CompressionZstd, + }, + { + FrameRestartFlags: pkg.RestartCodecs, + MaxUncompressedFrameByteSize: 500, + }, +} diff --git a/go/otel/otelstef/metricswriter_test.go b/go/otel/otelstef/metricswriter_test.go index 538e0d3c..b3ba05b3 100644 --- a/go/otel/otelstef/metricswriter_test.go +++ b/go/otel/otelstef/metricswriter_test.go @@ -43,30 +43,6 @@ func genMetricsRecords(random *rand.Rand, schem *schema.Schema) (records []Metri func testMetricsWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { retVal = true - opts := []pkg.WriterOptions{ - {}, - {Compression: pkg.CompressionZstd}, - {MaxUncompressedFrameByteSize: 500}, - {MaxTotalDictSize: 500}, - { - Compression: pkg.CompressionZstd, - MaxUncompressedFrameByteSize: 500, - MaxTotalDictSize: 500, - }, - {FrameRestartFlags: pkg.RestartDictionaries}, - {FrameRestartFlags: pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, - { - FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, - Compression: pkg.CompressionZstd, - }, - { - FrameRestartFlags: pkg.RestartCodecs, - MaxUncompressedFrameByteSize: 500, - }, - } - random := rand.New(rand.NewPCG(seed, 0)) // Load the schema from the allSchemaContent variable. @@ -83,7 +59,7 @@ func testMetricsWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { } wireSchema := schema.NewWireSchema(schem, "Metrics") - for _, opt := range opts { + for _, opt := range testWriterOpts { t.Run( "", func(t *testing.T) { succeeded := false @@ -95,10 +71,11 @@ func testMetricsWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { }() // Write data according to (possibly modified) schema - opt.Schema = &wireSchema + optCpy := opt + optCpy.Schema = &wireSchema buf := &pkg.MemChunkWriter{} - writer, err := NewMetricsWriter(buf, opt) + writer, err := NewMetricsWriter(buf, optCpy) require.NoError(t, err, "seed %v", seed) // Generate records pseudo-randomly @@ -163,3 +140,47 @@ func TestMetricsWriteRead(t *testing.T) { succeeded = testMetricsWriteReadSeed(t, seed) } + +func FuzzMetricsReader(f *testing.F) { + f.Add([]byte("")) + + random := rand.New(rand.NewPCG(0, 0)) + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(f, err) + + for _, opt := range testWriterOpts { + for i := 0; i <= 3; i++ { + buf := &pkg.MemChunkWriter{} + writer, err := NewMetricsWriter(buf, opt) + require.NoError(f, err) + + recCount := (1 << (2 * i)) - 1 + for range recCount { + limiter := &mutateRandomLimiter{} + writer.Record.mutateRandom(random, schem, limiter) + err = writer.Write() + require.NoError(f, err) + } + + err = writer.Flush() + require.NoError(f, err) + + f.Add(buf.Bytes()) + } + } + + f.Fuzz( + func(t *testing.T, data []byte) { + reader, err := NewMetricsReader(bytes.NewBuffer(data)) + if err != nil { + return + } + for { + err = reader.Read(pkg.ReadOptions{}) + if err != nil { + break + } + } + }, + ) +} diff --git a/go/otel/otelstef/spanswriter_test.go b/go/otel/otelstef/spanswriter_test.go index fa6bbd87..fa6c6575 100644 --- a/go/otel/otelstef/spanswriter_test.go +++ b/go/otel/otelstef/spanswriter_test.go @@ -43,30 +43,6 @@ func genSpansRecords(random *rand.Rand, schem *schema.Schema) (records []Spans) func testSpansWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { retVal = true - opts := []pkg.WriterOptions{ - {}, - {Compression: pkg.CompressionZstd}, - {MaxUncompressedFrameByteSize: 500}, - {MaxTotalDictSize: 500}, - { - Compression: pkg.CompressionZstd, - MaxUncompressedFrameByteSize: 500, - MaxTotalDictSize: 500, - }, - {FrameRestartFlags: pkg.RestartDictionaries}, - {FrameRestartFlags: pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, - {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, - { - FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, - Compression: pkg.CompressionZstd, - }, - { - FrameRestartFlags: pkg.RestartCodecs, - MaxUncompressedFrameByteSize: 500, - }, - } - random := rand.New(rand.NewPCG(seed, 0)) // Load the schema from the allSchemaContent variable. @@ -83,7 +59,7 @@ func testSpansWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { } wireSchema := schema.NewWireSchema(schem, "Spans") - for _, opt := range opts { + for _, opt := range testWriterOpts { t.Run( "", func(t *testing.T) { succeeded := false @@ -95,10 +71,11 @@ func testSpansWriteReadSeed(t *testing.T, seed uint64) (retVal bool) { }() // Write data according to (possibly modified) schema - opt.Schema = &wireSchema + optCpy := opt + optCpy.Schema = &wireSchema buf := &pkg.MemChunkWriter{} - writer, err := NewSpansWriter(buf, opt) + writer, err := NewSpansWriter(buf, optCpy) require.NoError(t, err, "seed %v", seed) // Generate records pseudo-randomly @@ -163,3 +140,47 @@ func TestSpansWriteRead(t *testing.T) { succeeded = testSpansWriteReadSeed(t, seed) } + +func FuzzSpansReader(f *testing.F) { + f.Add([]byte("")) + + random := rand.New(rand.NewPCG(0, 0)) + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(f, err) + + for _, opt := range testWriterOpts { + for i := 0; i <= 3; i++ { + buf := &pkg.MemChunkWriter{} + writer, err := NewSpansWriter(buf, opt) + require.NoError(f, err) + + recCount := (1 << (2 * i)) - 1 + for range recCount { + limiter := &mutateRandomLimiter{} + writer.Record.mutateRandom(random, schem, limiter) + err = writer.Write() + require.NoError(f, err) + } + + err = writer.Flush() + require.NoError(f, err) + + f.Add(buf.Bytes()) + } + } + + f.Fuzz( + func(t *testing.T, data []byte) { + reader, err := NewSpansReader(bytes.NewBuffer(data)) + if err != nil { + return + } + for { + err = reader.Read(pkg.ReadOptions{}) + if err != nil { + break + } + } + }, + ) +} diff --git a/stefc/templates/go/common.go.tmpl b/stefc/templates/go/common.go.tmpl index 0fb1132c..34cceb4f 100644 --- a/stefc/templates/go/common.go.tmpl +++ b/stefc/templates/go/common.go.tmpl @@ -4,6 +4,7 @@ import ( "fmt" "math" + "github.com/splunk/stef/go/pkg" "github.com/splunk/stef/go/pkg/schema" ) @@ -90,3 +91,27 @@ type mutateRandomLimiter struct { objectCount int elemCount int } + +var testWriterOpts []pkg.WriterOptions = []pkg.WriterOptions{ + {}, + {Compression: pkg.CompressionZstd}, + {MaxUncompressedFrameByteSize: 500}, + {MaxTotalDictSize: 500}, + { + Compression: pkg.CompressionZstd, + MaxUncompressedFrameByteSize: 500, + MaxTotalDictSize: 500, + }, + {FrameRestartFlags: pkg.RestartDictionaries}, + {FrameRestartFlags: pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs}, + {FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd}, + { + FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, + Compression: pkg.CompressionZstd, + }, + { + FrameRestartFlags: pkg.RestartCodecs, + MaxUncompressedFrameByteSize: 500, + }, +} diff --git a/stefc/templates/go/writer_test.go.tmpl b/stefc/templates/go/writer_test.go.tmpl index 0ee62865..6e60e8aa 100644 --- a/stefc/templates/go/writer_test.go.tmpl +++ b/stefc/templates/go/writer_test.go.tmpl @@ -42,30 +42,6 @@ func gen{{.StructName}}Records(random *rand.Rand, schem *schema.Schema) (records func test{{.StructName}}WriteReadSeed(t *testing.T, seed uint64) (retVal bool) { retVal = true - opts := []pkg.WriterOptions{ - {}, - { Compression: pkg.CompressionZstd }, - { MaxUncompressedFrameByteSize: 500 }, - { MaxTotalDictSize: 500 }, - { - Compression: pkg.CompressionZstd, - MaxUncompressedFrameByteSize: 500, - MaxTotalDictSize: 500, - }, - { FrameRestartFlags: pkg.RestartDictionaries }, - { FrameRestartFlags: pkg.RestartCodecs }, - { FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs }, - { FrameRestartFlags: pkg.RestartCompression, Compression: pkg.CompressionZstd }, - { - FrameRestartFlags: pkg.RestartDictionaries | pkg.RestartCodecs | pkg.RestartCompression, - Compression: pkg.CompressionZstd, - }, - { - FrameRestartFlags: pkg.RestartCodecs, - MaxUncompressedFrameByteSize: 500, - }, - } - random := rand.New(rand.NewPCG(seed, 0)) // Load the schema from the allSchemaContent variable. @@ -82,7 +58,7 @@ func test{{.StructName}}WriteReadSeed(t *testing.T, seed uint64) (retVal bool) { } wireSchema := schema.NewWireSchema(schem, "{{.StructName}}") - for _, opt := range opts { + for _, opt := range testWriterOpts { t.Run( "", func(t *testing.T) { succeeded := false @@ -94,10 +70,11 @@ func test{{.StructName}}WriteReadSeed(t *testing.T, seed uint64) (retVal bool) { }() // Write data according to (possibly modified) schema - opt.Schema = &wireSchema + optCpy := opt + optCpy.Schema = &wireSchema buf := &pkg.MemChunkWriter{} - writer, err := New{{.StructName}}Writer(buf, opt) + writer, err := New{{.StructName}}Writer(buf, optCpy) require.NoError(t, err, "seed %v", seed) // Generate records pseudo-randomly @@ -162,3 +139,47 @@ func Test{{.StructName}}WriteRead(t *testing.T) { succeeded = test{{.StructName}}WriteReadSeed(t, seed) } + +func Fuzz{{.StructName}}Reader(f *testing.F) { + f.Add([]byte("")) + + random := rand.New(rand.NewPCG(0, 0)) + schem, err := idl.Parse([]byte(allSchemaContent), "") + require.NoError(f, err) + + for _, opt := range testWriterOpts { + for i := 0; i <= 3; i++ { + buf := &pkg.MemChunkWriter{} + writer, err := New{{.StructName}}Writer(buf, opt) + require.NoError(f, err) + + recCount := (1 << (2*i)) - 1 + for range recCount { + limiter := &mutateRandomLimiter{} + writer.Record.mutateRandom(random, schem, limiter) + err = writer.Write() + require.NoError(f, err) + } + + err = writer.Flush() + require.NoError(f, err) + + f.Add(buf.Bytes()) + } + } + + f.Fuzz( + func(t *testing.T, data []byte) { + reader, err := New{{.StructName}}Reader(bytes.NewBuffer(data)) + if err != nil { + return + } + for { + err = reader.Read(pkg.ReadOptions{}) + if err != nil { + break + } + } + }, + ) +}