Cody littley/cryptosim metrics

go.mod

@@ -117,6 +117,11 @@ require (
 	pgregory.net/rapid v1.2.0
 )
 
+require (
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+)
+
 require (
 	github.com/quasilyte/go-ruleguard v0.3.18 // indirect
 	github.com/quasilyte/gogrep v0.0.0-20220828223005-86e4605de09f // indirect
@@ -334,6 +339,7 @@ require (
 	github.com/securego/gosec/v2 v2.13.1 // indirect
 	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/shirou/gopsutil/v3 v3.23.2
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sivchari/containedctx v1.0.2 // indirect
 	github.com/sivchari/nosnakecase v1.7.0 // indirect

go.sum

@@ -1637,6 +1637,7 @@ github.com/linxGnu/grocksdb v1.8.11/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXk
 github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4=
 github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCEtOM=
 github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
 github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
@@ -1905,6 +1906,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
 github.com/polyfloyd/go-errorlint v1.0.5 h1:AHB5JRCjlmelh9RrLxT9sgzpalIwwq4hqE8EkwIwKdY=
 github.com/polyfloyd/go-errorlint v1.0.5/go.mod h1:APVvOesVSAnne5SClsPxPdfvZTVDojXh1/G3qb5wjGI=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -2048,6 +2050,7 @@ github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAx
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
 github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQUExU=
 github.com/shirou/gopsutil/v3 v3.23.2/go.mod h1:gv0aQw33GLo3pG8SiWKiQrbDzbRY1K80RyZJ7V4Th1M=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=

sei-db/state_db/bench/cryptosim/README.md

@@ -50,3 +50,77 @@ The following features might be useful to add to this benchmark:
     - Metrics exported by the underlying DB
 - Prometheus/Grafana dashboards to visualize metrics
 - More exotic key access patterns
+
+
+# Setting Up Prometheus / Grafana
+
+To set up local prometheus/grafana instances, run the following. You must have docker installed.
+
+```
+./metrics/start-prometheus.sh
+./metrics/start-grafana.sh
+```
+
+Then, navigate to http://localhost:3000/ in a web browser to reach the grafana UI. Username and password are "admin".
+
+There is a pre-built dashboard containing visualizations for benchmark metrics in `metrics/dashboard.json` that 
+you can import into grafana.
+
+
+You can stop these services by killing their containers, or by running the following:
+
+```
+./metrics/stop-prometheus.sh
+./metrics/stop-grafana.sh
+```
+
+# Running in AWS
+
+To set up this benchmark on an AWS machine, perform the following steps:
+
+1. Clone the repo
+
+```
+git clone https://github.com/sei-protocol/sei-chain.git
+```
+
+2. Install dependencies
+
+```
+cd ./sei-chain/sei-db/state_db/bench/cryptosim
+./tools/setup-ubuntu.sh
+```
+
+3: Start Prometheus Server (optional)
+
+```
+cd ./sei-chain/sei-db/state_db/bench/cryptosim
+./metrics/start-prometheus.sh
+```
+
+3. Start the Benchmark
+
+Optional: start a tmux session (the install script installs tmux). This will allow the benchmark to run
+even if your connection is interrupted.
+
+```
+cd sei-chain/sei-db/state_db/bench/cryptosim
+./run.sh ./config/basic-config.json
+```
+
+4: Connect Local Grafana to Remote Prometheus (optional)
+
+Before taking this step, you will need to set up SSH access to the remote VM. The easiest way to do this is to
+copy your public ssh key into the remote machine's `~/.ssh/authorized_keys` file.
+
+You will want the prometheus container running on your remote machine, and the grafana container running locally.
+Make sure you don't have a local prometheus container running.
+
+Then, initiate an ssh connection with the remote machine using this command:
+
+```
+ssh -L 9091:localhost:9091 user@remote-host
+```
+
+As long as this connection remains open, your local grafana instance will be able to talk to the remote prometheus
+instance over the SSH tunnel.

sei-db/state_db/bench/cryptosim/config/basic-config.json

@@ -2,7 +2,7 @@
   "Comment": "Basic configuration for the cryptosim benchmark. Intended for basic correctness/sanity testing.",
   "Backend": "FlatKV",
   "BlocksPerCommit": 32,
-  "CannedRandomSize": 1073741824,  
+  "CannedRandomSize": 1073741824,
   "ConstantThreadCount": 0,
   "ConsoleUpdateIntervalSeconds": 1,
   "ConsoleUpdateIntervalTransactions": 1000000,
@@ -16,12 +16,15 @@
   "HotErc20ContractProbability": 0.5,
   "HotErc20ContractSetSize": 100,
   "MinimumNumberOfAccounts": 1000000,
+  "MetricsAddr": ":9090",
   "MinimumNumberOfErc20Contracts": 10000,
   "NewAccountProbability": 0.001,
   "PaddedAccountSize": 69,
   "Seed": 1337,
   "SetupUpdateIntervalCount": 100000,
   "ThreadsPerCore": 2.0,
   "TransactionsPerBlock": 1024,
-  "MaxRuntimeSeconds": 0
+  "MaxRuntimeSeconds": 0,
+  "TransactionMetricsSampleRate": 0.001,
+  "BackgroundMetricsScrapeInterval": 60
 }

sei-db/state_db/bench/cryptosim/config/large.json

@@ -0,0 +1,6 @@
+{
+    "Comment": "A large simulation. This is the largest sane simulation for local testing.",
+    "DataDir": "data",
+    "MinimumNumberOfAccounts": 100000000
+}
+

sei-db/state_db/bench/cryptosim/cryptosim.go

@@ -56,6 +56,9 @@ type CryptoSim struct {
 
 	// The index of the next executor to receive a transaction.
 	nextExecutorIndex int
+
+	// The metrics for the benchmark.
+	metrics *CryptosimMetrics
 }
 
 // Creates a new cryptosim benchmark runner.
@@ -68,15 +71,22 @@ func NewCryptoSim(
 		return nil, fmt.Errorf("invalid config: %w", err)
 	}
 
+	ctx, cancel := context.WithCancel(ctx)
+	metrics := NewCryptosimMetrics(ctx, config)
+	// Server start deferred until after DataGenerator loads DB state and sets gauges,
+	// avoiding rate() spikes when restarting with a preserved DB.
+
 	dataDir, err := resolveAndCreateDataDir(config.DataDir)
 	if err != nil {
-		return nil, err
+		cancel()
+		return nil, fmt.Errorf("failed to resolve and create data directory: %w", err)
 	}
 
 	fmt.Printf("Running cryptosim benchmark from data directory: %s\n", dataDir)
 
 	db, err := wrappers.NewDBImpl(config.Backend, dataDir)
 	if err != nil {
+		cancel()
 		return nil, fmt.Errorf("failed to create database: %w", err)
 	}
 
@@ -87,19 +97,16 @@ func NewCryptoSim(
 
 	start := time.Now()
 
-	ctx, cancel := context.WithCancel(ctx)
-
-	database := NewDatabase(config, db)
+	database := NewDatabase(config, db, metrics)
 
-	dataGenerator, err := NewDataGenerator(config, database, rand)
+	dataGenerator, err := NewDataGenerator(config, database, rand, metrics)
 	if err != nil {
 		cancel()
 		if closeErr := db.Close(); closeErr != nil {
 			fmt.Printf("failed to close database during error recovery: %v\n", closeErr)
 		}
 		return nil, fmt.Errorf("failed to create data generator: %w", err)
 	}
-
 	threadCount := int(config.ThreadsPerCore)*runtime.NumCPU() + config.ConstantThreadCount
 	if threadCount < 1 {
 		threadCount = 1
@@ -109,7 +116,7 @@ func NewCryptoSim(
 	executors := make([]*TransactionExecutor, threadCount)
 	for i := 0; i < threadCount; i++ {
 		executors[i] = NewTransactionExecutor(
-			ctx, database, dataGenerator.FeeCollectionAddress(), config.ExecutorQueueSize)
+			ctx, cancel, database, dataGenerator.FeeCollectionAddress(), config.ExecutorQueueSize, metrics)
 	}
 
 	c := &CryptoSim{
@@ -123,6 +130,7 @@ func NewCryptoSim(
 		dataGenerator:                     dataGenerator,
 		database:                          database,
 		executors:                         executors,
+		metrics:                           metrics,
 	}
 
 	database.SetFlushFunc(c.flushExecutors)
@@ -134,6 +142,7 @@ func NewCryptoSim(
 
 	c.database.ResetTransactionCount()
 	c.startTimestamp = time.Now()
+	c.metrics.StartServer(config.MetricsAddr)
 
 	go c.run()
 	return c, nil
@@ -184,6 +193,7 @@ func (c *CryptoSim) setupAccounts() error {
 			return fmt.Errorf("failed to maybe commit batch: %w", err)
 		}
 		if finalized {
+			c.metrics.SetTotalNumberOfAccounts(c.dataGenerator.NextAccountID())
 			c.dataGenerator.ReportFinalizeBlock()
 		}
 
@@ -202,6 +212,7 @@ func (c *CryptoSim) setupAccounts() error {
 	if err != nil {
 		return fmt.Errorf("failed to finalize block: %w", err)
 	}
+	c.metrics.SetTotalNumberOfAccounts(c.dataGenerator.NextAccountID())
 	c.dataGenerator.ReportFinalizeBlock()
 
 	fmt.Printf("There are now %s accounts in the database.\n", int64Commas(c.dataGenerator.NextAccountID()))
@@ -245,6 +256,7 @@ func (c *CryptoSim) setupErc20Contracts() error {
 		}
 		if finalized {
 			c.dataGenerator.ReportFinalizeBlock()
+			c.metrics.SetTotalNumberOfERC20Contracts(c.dataGenerator.NextErc20ContractID())
 		}
 
 		if c.dataGenerator.NextErc20ContractID()%c.config.SetupUpdateIntervalCount == 0 {
@@ -266,6 +278,7 @@ func (c *CryptoSim) setupErc20Contracts() error {
 		return fmt.Errorf("failed to finalize block: %w", err)
 	}
 	c.dataGenerator.ReportFinalizeBlock()
+	c.metrics.SetTotalNumberOfERC20Contracts(c.dataGenerator.NextErc20ContractID())
 
 	fmt.Printf("There are now %s simulated ERC20 contracts in the database.\n",
 		int64Commas(c.dataGenerator.NextErc20ContractID()))
@@ -280,6 +293,8 @@ func (c *CryptoSim) run() {
 
 	haltTime := time.Now().Add(time.Duration(c.config.MaxRuntimeSeconds) * time.Second)
 
+	c.metrics.SetMainThreadPhase("executing")
+
 	for {
 		select {
 		case <-c.ctx.Done():
@@ -293,6 +308,7 @@ func (c *CryptoSim) run() {
 			txn, err := BuildTransaction(c.dataGenerator)
 			if err != nil {
 				fmt.Printf("\nfailed to build transaction: %v\n", err)
+				c.cancel()
 				continue
 			}
 
@@ -303,6 +319,8 @@ func (c *CryptoSim) run() {
 				c.dataGenerator.NextAccountID(), c.dataGenerator.NextErc20ContractID())
 			if err != nil {
 				fmt.Printf("error finalizing block: %v\n", err)
+				c.cancel()
+				continue
 			}
 			if finalized {
 				c.dataGenerator.ReportFinalizeBlock()

sei-db/state_db/bench/cryptosim/cryptosim_config.go

@@ -108,6 +108,16 @@ type CryptoSimConfig struct {
 
 	// The amount of time to run the benchmark for. If 0, the benchmark will run until it is stopped.
 	MaxRuntimeSeconds int
+
+	// Address for the Prometheus metrics HTTP server (e.g. ":9090"). If empty, metrics are disabled.
+	MetricsAddr string
+
+	// The probability of capturing detailed metrics about a transaction. Should be a value between 0.0 and 1.0.
+	TransactionMetricsSampleRate float64
+
+	// How often (in seconds) to scrape background metrics (data dir size, process I/O).
+	// If 0, background metrics are disabled.
+	BackgroundMetricsScrapeInterval int
 }
 
 // Returns the default configuration for the cryptosim benchmark.
@@ -140,6 +150,9 @@ func DefaultCryptoSimConfig() *CryptoSimConfig {
 		ConstantThreadCount:               0,
 		ExecutorQueueSize:                 64,
 		MaxRuntimeSeconds:                 0,
+		MetricsAddr:                       ":9090",
+		TransactionMetricsSampleRate:      0.001,
+		BackgroundMetricsScrapeInterval:   60,
 	}
 }
 
@@ -202,6 +215,13 @@ func (c *CryptoSimConfig) Validate() error {
 	if c.MaxRuntimeSeconds < 0 {
 		return fmt.Errorf("MaxRuntimeSeconds must be at least 0 (got %d)", c.MaxRuntimeSeconds)
 	}
+	if c.TransactionMetricsSampleRate < 0 || c.TransactionMetricsSampleRate > 1 {
+		return fmt.Errorf("TransactionMetricsSampleRate must be in [0, 1] (got %f)", c.TransactionMetricsSampleRate)
+	}
+	if c.BackgroundMetricsScrapeInterval < 0 {
+		return fmt.Errorf("BackgroundMetricsScrapeInterval must be non-negative (got %d)", c.BackgroundMetricsScrapeInterval)
+	}
+
 	return nil
 }