Merge pull request #399 from canopy-network/custom-compression

feat: custom compression algorithm

Author: rem1niscence

lib/config.go

@@ -279,6 +279,7 @@ type StoreConfig struct {
 	LSSCompactionInterval uint64 `json:"lssCompactionInterval"` // interval for compacting latest store data
 	BackupDirectory       string `json:"backupDirectory"`       // directory where backups of the database are stored
 	BackupInterval        uint64 `json:"backupInterval"`        // interval in blocks for creating backups of the database (0 to disable automatic backups)
+	CompressionProfile    string `json:"compressionProfile"`    // the pebbledb compression profile to use.
 }
 
 // DefaultDataDirPath() is $USERHOME/.canopy
@@ -306,6 +307,7 @@ func DefaultStoreConfig() StoreConfig {
 		LSSCompactionInterval: uint64(rand.Int32N(101) + 500),            // clean every 500-600 blocks (random)
 		BackupDirectory:       path.Join(DefaultDataDirPath(), "backup"), // backup directory name
 		BackupInterval:        0,                                         // backups disabled by default
+		CompressionProfile:    "zstd",
 	}
 }
 

lib/store.go

@@ -103,7 +103,7 @@ type ProveStoreI interface {
 // IteratorI defines an interface for iterating over key-value pairs in a data store
 type IteratorI interface {
 	Valid() bool           // if the item the iterator is pointing at is valid
-	Next()                 // move to next item
+	Next()                 // move to next itema
 	Key() (key []byte)     // retrieve key
 	Value() (value []byte) // retrieve value
 	Close()                // close the iterator when done, ensuring proper resource management

store/store.go

@@ -97,7 +97,9 @@ func NewStore(config lib.Config, path string, metrics *lib.Metrics, log lib.Logg
 		BlockSize:      64 << 10, // 64 KB data blocks
 		IndexBlockSize: 32 << 10, // 32 KB index blocks
 		Compression: func() *sstable.CompressionProfile {
-			return sstable.ZstdCompression // biggest compression at the expense of more CPU resources
+			profile := getCompressionProfile(config.CompressionProfile)
+			log.Infof("Using %s compression for sstables", profile.Name)
+			return profile
 		},
 	}
 	db, err := pebble.Open(path, &pebble.Options{

store/versioned_store.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"math"
+	"strings"
 
 	"github.com/canopy-network/canopy/lib"
 	"github.com/cockroachdb/pebble/v2"
@@ -641,3 +642,23 @@ func newTargetWindowFilter(low, high uint64) sstable.BlockPropertyFilter {
 		nil,
 	)
 }
+
+// getCompressionProfile returns the compression profile (algorithm) to use for the versioned store
+func getCompressionProfile(profile string) *sstable.CompressionProfile {
+	switch strings.ToLower(profile) {
+	case "zstd":
+		return sstable.ZstdCompression
+	case "snappy":
+		return sstable.SnappyCompression
+	case "nocompression":
+		return sstable.NoCompression
+	case "fastest":
+		return sstable.FastestCompression
+	case "balanced":
+		return sstable.BalancedCompression
+	case "good":
+		return sstable.GoodCompression
+	default:
+		return sstable.ZstdCompression
+	}
+}

store/versioned_store_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/canopy-network/canopy/lib"
+	"github.com/cockroachdb/pebble/v2/sstable"
 	"github.com/stretchr/testify/require"
 )
 
@@ -65,3 +66,24 @@ func TestMakeVersionedKeyConcurrent(t *testing.T) {
 	require.False(t, failed.Load(), "concurrent makeVersionedKey produced malformed keys")
 }
 
+func TestGetCompressionProfile(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected *sstable.CompressionProfile
+	}{
+		{"zstd", sstable.ZstdCompression},
+		{"ZSTD", sstable.ZstdCompression},
+		{"snappy", sstable.SnappyCompression},
+		{"fastest", sstable.FastestCompression},
+		{"balanced", sstable.BalancedCompression},
+		{"good", sstable.GoodCompression},
+		{"noCompression", sstable.NoCompression},
+		{"unknown", sstable.ZstdCompression},
+		{"", sstable.ZstdCompression},
+	}
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			require.Equal(t, tt.expected, getCompressionProfile(tt.input))
+		})
+	}
+}