feat: introduce schemas (#1064)

* feat: introduce schemas

* feat: plug schema on writes (#1065)

* feat: plug schema on writes

* feat: Chart of accounts & tx validation (#1109)

* wip chart of account & serialization

* tx validation

* improve structure

* fix test

* more tests & fixes

* cleanup & improve coverage

* remove rules prototype

* force schemaVersion if ledger has one

* cleanup & better naming

* remove useless if, rename regex

* change segment prop prefix

* add missing test & cleanup leftover errors

* missed pre-commit

* address review suggestions

- used strings.HasPrefix to avoid potential panics
- group var definitions
- validate .pattern type & regex
- fix .self validation `{ }`

* improve error message for forbidden keys

* fix permissive regexes

* table-driven tests & t.Parallel

* fix api pattern examples

* review fixes

- Use log validation method instead of ad-hoc controller check
- Handle errors: non-specified schema, already-existing schema version, missing chart
- Add e2e unspecified schema check

* fix test

---------

Co-authored-by: Alix Bott <bott.alix@gmail.com>

* improve error messages (#1148)

* remove special handling of 'world' in charts

* feat: Chart of accounts: default metadata (#1164)

* default metadata on upsert account

* use global schemaVersion on bulks

* fix migrations

* fix missing units in schema otel histograms

* consistency: rename update->insert

* 400->409 for SchemaAlreadyExist

* tweak comment

* fix pagination test, add `previous` to v2schemacursor

* tweak test

* fix: read schema (#1180)

* improve naming consistency

* skip marshaling empty rules

* fix marshal

* pre-commit

* throw error on root .pattern + allow patternless variable segments

* throw error when account fields are used on non-accounts

* fix migration number

* fix

* schema enforcement mode

* address review comments

- pass schema to createTransaction
- schemaVersion in BulkingOptions
- use Store.newScopedSelect
- remove the errors.Unwraps

---------

Co-authored-by: Alix Bott <bott.alix@gmail.com>
Co-authored-by: David Ragot <35502263+Dav-14@users.noreply.github.com>

Author: 3 people

cmd/config.go

@@ -8,14 +8,17 @@ import (
 	"github.com/robfig/cron/v3"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/formancehq/ledger/internal/controller/ledger"
 )
 
 type commonConfig struct {
-	NumscriptInterpreter        bool     `mapstructure:"experimental-numscript-interpreter"`
-	NumscriptInterpreterFlags   []string `mapstructure:"experimental-numscript-interpreter-flags"`
-	ExperimentalFeaturesEnabled bool     `mapstructure:"experimental-features"`
-	ExperimentalExporters       bool     `mapstructure:"experimental-exporters"`
-	SemconvMetricsNames         bool     `mapstructure:"semconv-metrics-names"`
+	NumscriptInterpreter        bool                         `mapstructure:"experimental-numscript-interpreter"`
+	NumscriptInterpreterFlags   []string                     `mapstructure:"experimental-numscript-interpreter-flags"`
+	ExperimentalFeaturesEnabled bool                         `mapstructure:"experimental-features"`
+	ExperimentalExporters       bool                         `mapstructure:"experimental-exporters"`
+	SemconvMetricsNames         bool                         `mapstructure:"semconv-metrics-names"`
+	SchemaEnforcementMode       ledger.SchemaEnforcementMode `mapstructure:"schema-enforcement-mode"`
 }
 
 func decodeCronSchedule(sourceType, destType reflect.Type, value any) (any, error) {

cmd/docs_events.go

@@ -36,6 +36,7 @@ func NewDocEventsCommand() *cobra.Command {
 				events.DeletedMetadata{},
 				events.SavedMetadata{},
 				events.RevertedTransaction{},
+				events.InsertedSchema{},
 			} {
 				schema := jsonschema.Reflect(o)
 				data, err := json.MarshalIndent(schema, "", "  ")

cmd/serve.go

@@ -65,10 +65,11 @@ const (
 	BulkMaxSizeFlag            = "bulk-max-size"
 	BulkParallelFlag           = "bulk-parallel"
 
-	DefaultPageSizeFlag = "default-page-size"
-	MaxPageSizeFlag     = "max-page-size"
-	WorkerEnabledFlag   = "worker"
-	SemconvMetricsNames = "semconv-metrics-names"
+	DefaultPageSizeFlag   = "default-page-size"
+	MaxPageSizeFlag       = "max-page-size"
+	WorkerEnabledFlag     = "worker"
+	SemconvMetricsNames   = "semconv-metrics-names"
+	SchemaEnforcementMode = "schema-enforcement-mode"
 )
 
 func NewServeCommand() *cobra.Command {
@@ -112,7 +113,8 @@ func NewServeCommand() *cobra.Command {
 						MaxRetry: 10,
 						Delay:    time.Millisecond * 100,
 					},
-					EnableFeatures: cfg.ExperimentalFeaturesEnabled,
+					EnableFeatures:        cfg.ExperimentalFeaturesEnabled,
+					SchemaEnforcementMode: cfg.commonConfig.SchemaEnforcementMode,
 				}),
 				bus.NewFxModule(),
 				ballast.Module(cfg.BallastSizeInBytes),
@@ -187,6 +189,7 @@ func NewServeCommand() *cobra.Command {
 	cmd.Flags().StringSlice(NumscriptInterpreterFlagsToPass, nil, "Feature flags to pass to the experimental numscript interpreter")
 	cmd.Flags().String(WorkerGRPCAddressFlag, "localhost:8081", "GRPC address")
 	cmd.Flags().Bool(SemconvMetricsNames, false, "Use semconv metrics names (recommended)")
+	cmd.Flags().String(SchemaEnforcementMode, "audit", "Schema enforcement mode. Values: `audit`, `strict`")
 
 	addWorkerFlags(cmd)
 	bunconnect.AddFlags(cmd.Flags())