Introduce new Vault limits for all ocr3.1 config fields

pkg/settings/cresettings/README.md

@@ -181,6 +181,16 @@ flowchart
         VaultIdentifierNamespaceSizeLimit{{VaultIdentifierNamespaceSizeLimit}}:::bound
         VaultPluginBatchSizeLimit{{VaultPluginBatchSizeLimit}}:::bound
         VaultRequestBatchSizeLimit{{VaultRequestBatchSizeLimit}}:::bound
+        VaultMaxQuerySizeLimit{{VaultMaxQuerySizeLimit}}:::bound
+        VaultMaxObservationSizeLimit{{VaultMaxObservationSizeLimit}}:::bound
+        VaultMaxReportsPlusPrecursorSizeLimit{{VaultMaxReportsPlusPrecursorSizeLimit}}:::bound
+        VaultMaxReportSizeLimit{{VaultMaxReportSizeLimit}}:::bound
+        VaultMaxReportCount{{VaultMaxReportCount}}:::bound
+        VaultMaxKeyValueModifiedKeysPlusValuesSizeLimit{{VaultMaxKeyValueModifiedKeysPlusValuesSizeLimit}}:::bound
+        VaultMaxKeyValueModifiedKeys{{VaultMaxKeyValueModifiedKeys}}:::bound
+        VaultMaxBlobPayloadSizeLimit{{VaultMaxBlobPayloadSizeLimit}}:::bound
+        VaultMaxPerOracleUnexpiredBlobCumulativePayloadSizeLimit{{VaultMaxPerOracleUnexpiredBlobCumulativePayloadSizeLimit}}:::bound
+        VaultMaxPerOracleUnexpiredBlobCount{{VaultMaxPerOracleUnexpiredBlobCount}}:::bound
         PerOwner.VaultSecretsLimit{{PerOwner.VaultSecretsLimit}}:::bound
     end
 

pkg/settings/cresettings/defaults.json

@@ -11,8 +11,18 @@
 	"VaultIdentifierKeySizeLimit": "64b",
 	"VaultIdentifierOwnerSizeLimit": "64b",
 	"VaultIdentifierNamespaceSizeLimit": "64b",
-	"VaultPluginBatchSizeLimit": "20",
+	"VaultPluginBatchSizeLimit": "10",
 	"VaultRequestBatchSizeLimit": "10",
+	"VaultMaxQuerySizeLimit": "102.4kb",
+	"VaultMaxObservationSizeLimit": "2mb",
+	"VaultMaxReportsPlusPrecursorSizeLimit": "2mb",
+	"VaultMaxReportSizeLimit": "2mb",
+	"VaultMaxReportCount": "10",
+	"VaultMaxKeyValueModifiedKeysPlusValuesSizeLimit": "1.468006mb",
+	"VaultMaxKeyValueModifiedKeys": "300",
+	"VaultMaxBlobPayloadSizeLimit": "25.6kb",
+	"VaultMaxPerOracleUnexpiredBlobCumulativePayloadSizeLimit": "31.45728mb",
+	"VaultMaxPerOracleUnexpiredBlobCount": "1000",
 	"PerOrg": {
 		"ZeroBalancePruningTimeout": "24h0m0s"
 	},

pkg/settings/cresettings/defaults.toml

@@ -10,8 +10,18 @@ VaultShareSizeLimit = '600b'
 VaultIdentifierKeySizeLimit = '64b'
 VaultIdentifierOwnerSizeLimit = '64b'
 VaultIdentifierNamespaceSizeLimit = '64b'
-VaultPluginBatchSizeLimit = '20'
+VaultPluginBatchSizeLimit = '10'
 VaultRequestBatchSizeLimit = '10'
+VaultMaxQuerySizeLimit = '102.4kb'
+VaultMaxObservationSizeLimit = '2mb'
+VaultMaxReportsPlusPrecursorSizeLimit = '2mb'
+VaultMaxReportSizeLimit = '2mb'
+VaultMaxReportCount = '10'
+VaultMaxKeyValueModifiedKeysPlusValuesSizeLimit = '1.468006mb'
+VaultMaxKeyValueModifiedKeys = '300'
+VaultMaxBlobPayloadSizeLimit = '25.6kb'
+VaultMaxPerOracleUnexpiredBlobCumulativePayloadSizeLimit = '31.45728mb'
+VaultMaxPerOracleUnexpiredBlobCount = '1000'
 
 [PerOrg]
 ZeroBalancePruningTimeout = '24h0m0s'

pkg/settings/cresettings/settings.go

@@ -67,10 +67,50 @@ var Default = Schema{
 	VaultIdentifierKeySizeLimit:       Size(64 * config.Byte),
 	VaultIdentifierOwnerSizeLimit:     Size(64 * config.Byte),
 	VaultIdentifierNamespaceSizeLimit: Size(64 * config.Byte),
-	VaultPluginBatchSizeLimit:         Int(20),
+	VaultPluginBatchSizeLimit:         Int(10),
 	VaultRequestBatchSizeLimit:        Int(10),
 	VaultShareSizeLimit:               Size(600 * config.Byte),
 
+	VaultMaxQuerySizeLimit: Size(102400 * config.Byte),
+	// Back of the envelope calculation:
+	// - An item can contain 2KB of ciphertext, 192 bytes of metadata (key, owner, namespace),
+	// a UUID (16 bytes) plus some overhead = ~2.5KB per item
+	// There can be 10 such items in a request, and 20 per batch, so 2.5KB * 10 * 20 = 500KB
+	// However as a buffer, setting the next 3 fields to 2 mb.
+	VaultMaxObservationSizeLimit:          Size(2 * config.MByte),
+	VaultMaxReportsPlusPrecursorSizeLimit: Size(2 * config.MByte),
+	VaultMaxReportSizeLimit:               Size(2 * config.MByte),
+	VaultMaxReportCount:                   Int(10),
+	// assumption for largest item:
+	// create request with the maximum ciphertext length:
+	// - 192 bytes (sum of MaxIdentifierKeyLengthBytes + MaxIdentifierOwnerLengthBytes + MaxIdentifierNamespaceLengthBytes)
+	// - 2048 bytes (MaxCiphertextLengthBytes)
+	// = ~2240 bytes for an item
+	// There are 10 items per request (separate vault setting), 10 request per batch (BatchSize)
+	// i.e. ~224 KB per batch
+	// For a batch we will write:
+	// - a secret + metadata record per item
+	//   - the secrets are 224 KB total
+	//   - the metadata is a list of secret identifiers,
+	//     there are a maximum of 100 secrets per owner (MaxSecretsPerOwner)
+	//     i.e. 192 bytes * 100 = ~19.2 KB
+	// - the pending queue
+	//   - 10 requests in the pending queue, each request is ~22.4Kb = ~22.4 KB
+	//   - an index record =  8bytes
+	// - total = ~224 KB + ~19.2 KB + ~224 KB + 8 bytes = ~467.2 KB
+	// Setting to 1.4MB to allow for some buffer.
+	VaultMaxKeyValueModifiedKeysPlusValuesSizeLimit: Size(1468006 * config.Byte),
+	// 10 batch size * 10 items per batch * 2 records modified per item (secret + metadata record)
+	// plus 10 batchsize items in the pending queue + 1 index record
+	// = 211 total.
+	// plus some buffer.
+	VaultMaxKeyValueModifiedKeys: Int(300),
+	// Assuming a request is max 25KB, we add a bit of buffer to allow some room.
+	VaultMaxBlobPayloadSizeLimit: Size(25600 * config.Byte),
+	// Per docs, this should allow some additional buffer to allow for reaping time.
+	VaultMaxPerOracleUnexpiredBlobCumulativePayloadSizeLimit: Size(31457280 * config.Byte),
+	VaultMaxPerOracleUnexpiredBlobCount:                      Int(1000),
+
 	PerOrg: Orgs{
 		ZeroBalancePruningTimeout: Duration(24 * time.Hour),
 	},
@@ -185,6 +225,17 @@ type Schema struct {
 	VaultPluginBatchSizeLimit         Setting[int] `unit:"{request}"`
 	VaultRequestBatchSizeLimit        Setting[int] `unit:"{request}"`
 
+	VaultMaxQuerySizeLimit                                   Setting[config.Size]
+	VaultMaxObservationSizeLimit                             Setting[config.Size]
+	VaultMaxReportsPlusPrecursorSizeLimit                    Setting[config.Size]
+	VaultMaxReportSizeLimit                                  Setting[config.Size]
+	VaultMaxReportCount                                      Setting[int]
+	VaultMaxKeyValueModifiedKeysPlusValuesSizeLimit          Setting[config.Size]
+	VaultMaxKeyValueModifiedKeys                             Setting[int]
+	VaultMaxBlobPayloadSizeLimit                             Setting[config.Size]
+	VaultMaxPerOracleUnexpiredBlobCumulativePayloadSizeLimit Setting[config.Size]
+	VaultMaxPerOracleUnexpiredBlobCount                      Setting[int]
+
 	PerOrg      Orgs      `scope:"org"`
 	PerOwner    Owners    `scope:"owner"`
 	PerWorkflow Workflows `scope:"workflow"`