diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e309e8e155..d14beaf640 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,7 @@ jobs: - name: Run build run: ./scripts/release.sh if: steps.changed-files.outputs.any_changed == 'true' - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: "cronosd-tarball-${{ matrix.os }}" path: "*.tar.gz" @@ -119,7 +119,7 @@ jobs: set +e (git diff --no-ext-diff --exit-code) echo "name=changed::$?" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: steps.changes.outputs.changed == 1 with: name: gomod2nix.toml @@ -162,7 +162,7 @@ jobs: set +e (git diff --no-ext-diff --exit-code) echo "name=changed::$?" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: steps.changes.outputs.changed == 1 with: name: contracts_out diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml index 44a89c320e..06736c5ef2 100644 --- a/.github/workflows/sims.yml +++ b/.github/workflows/sims.yml @@ -76,7 +76,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Github API Request id: request - uses: octokit/request-action@v2.0.0 + uses: octokit/request-action@main if: github.event_name == 'issue_comment' with: route: ${{ github.event.issue.pull_request.url }} @@ -120,7 +120,7 @@ jobs: echo ${{ job.status }} > status_build.txt - name: Upload file status_build.txt as an artifact if: github.event_name == 'issue_comment' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pass_status_build path: status_build.txt @@ -146,7 +146,7 @@ jobs: echo ${{ job.status }} > status_install.txt - name: Upload file status_install.txt as an artifact if: github.event_name == 'issue_comment' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pass_status_install path: status_install.txt @@ -202,7 +202,7 @@ jobs: echo ${{ job.status }} > status_sim1.txt - name: Upload file status_sim1.txt as an artifact if: github.event_name == 'issue_comment' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pass_status_sim1 path: status_sim1.txt @@ -258,7 +258,7 @@ jobs: echo ${{ job.status }} > status_sim2.txt - name: Upload file status_sim2.txt as an artifact if: github.event_name == 'issue_comment' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pass_status_sim2 path: status_sim2.txt @@ -314,7 +314,7 @@ jobs: echo ${{ job.status }} > status_sim3.txt - name: Upload file status_sim3.txt as an artifact if: github.event_name == 'issue_comment' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pass_status_sim3 path: status_sim3.txt @@ -370,7 +370,7 @@ jobs: echo ${{ job.status }} > status_sim4.txt - name: Upload file status_sim4.txt as an artifact if: github.event_name == 'issue_comment' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pass_status_sim4 path: status_sim4.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cbf1248110..5cf9e04b20 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,7 +56,7 @@ jobs: - name: 'Tar debug files' if: failure() run: tar cfz debug_files.tar.gz -C "${TMPDIR-/tmp}/pytest-of-runner" . - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: failure() with: name: debug-files @@ -66,7 +66,7 @@ jobs: if: steps.changed-files.outputs.only_changed == 'false' run: | nix profile install ./nix#go_1_22 - go tool covdata textfmt -i=integration_tests/coverage -o profile.txt + go tool covdata textfmt -i=coverage -o profile.txt - name: Upload coverage report if: steps.changed-files.outputs.only_changed == 'false' uses: codecov/codecov-action@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cbf253448..af3bd4dfa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## UNRELEASED + +### Bug Fixes + +* [#1720](https://github.com/crypto-org-chain/cronos/pull/1720) Include the fix of performance regression after upgrade in iavl. +* [#1748](https://github.com/crypto-org-chain/cronos/pull/1748) Query with GetCFWithTS to compare both timestamp and key to avoid run fixdata multiple times. +* (versiondb) [#1751](https://github.com/crypto-org-chain/cronos/pull/1751) Add missing Destroy for read options to properly hold and release options reference. + +### Improvements + +* [#1747](https://github.com/crypto-org-chain/cronos/pull/1747) Skip batch initialization and flush when fixdata with dry-run. + *Nov 13, 2024* ## v1.3.4 diff --git a/Makefile b/Makefile index 03131ec304..378e740512 100644 --- a/Makefile +++ b/Makefile @@ -235,7 +235,7 @@ TESTS_TO_RUN ?= all run-integration-tests: @make gen-bindings-contracts - @nix-shell ./integration_tests/shell.nix --run ./scripts/run-integration-tests + @./scripts/run-integration-tests .PHONY: run-integration-tests diff --git a/integration_tests/cosmoscli.py b/integration_tests/cosmoscli.py index 71a5b64688..b74e17d8ef 100644 --- a/integration_tests/cosmoscli.py +++ b/integration_tests/cosmoscli.py @@ -1804,6 +1804,11 @@ def restore_versiondb(self, height, format=3, **kwargs): **kwargs, ) + def changeset_fixdata(self, versiondb_dir, dry_run=False): + return self.raw( + "changeset", "fixdata", versiondb_dir, "--dry-run" if dry_run else None + ) + def dump_snapshot(self, height, tarball, format=3): return self.raw( "snapshots", "dump", height, format, home=self.data_dir, output=tarball diff --git a/integration_tests/pytest.ini b/integration_tests/pytest.ini index b8fee6fed3..bcc1429d41 100644 --- a/integration_tests/pytest.ini +++ b/integration_tests/pytest.ini @@ -1,2 +1,3 @@ [pytest] addopts = --ignore=contracts +python_files = integration_tests/*.py diff --git a/integration_tests/test_upgrade.py b/integration_tests/test_upgrade.py index 723dd88fd9..9c0484786d 100644 --- a/integration_tests/test_upgrade.py +++ b/integration_tests/test_upgrade.py @@ -2,6 +2,7 @@ import shutil import stat import subprocess +import time from contextlib import contextmanager from datetime import datetime, timedelta from pathlib import Path @@ -274,6 +275,13 @@ def do_upgrade( do_upgrade("v1.3", target4, gas_prices=gas_prices) cli = c.cosmos_cli() + c.supervisorctl("stop", "cronos_777-1-node0") + time.sleep(3) + cli.changeset_fixdata(f"{c.base_dir}/node0/data/versiondb") + assert not cli.changeset_fixdata(f"{c.base_dir}/node0/data/versiondb", dry_run=True) + c.supervisorctl("start", "cronos_777-1-node0") + wait_for_port(ports.evmrpc_port(c.base_port(0))) + # check basic tx works wait_for_port(ports.evmrpc_port(c.base_port(0))) receipt = send_transaction( diff --git a/scripts/run-integration-tests b/scripts/run-integration-tests index c2278eb22b..6b91704584 100755 --- a/scripts/run-integration-tests +++ b/scripts/run-integration-tests @@ -9,7 +9,7 @@ echo "build test contracts" cd ../integration_tests/contracts HUSKY_SKIP_INSTALL=1 npm install npm run typechain -cd .. +cd ../.. TESTS_TO_RUN="${TESTS_TO_RUN:-all}" if [[ "$TESTS_TO_RUN" == "all" ]]; then echo "run all tests" @@ -18,4 +18,4 @@ else echo "run tests matching $TESTS_TO_RUN" cmd="pytest -vv -s -m '$TESTS_TO_RUN'" fi -nix-shell --run "$cmd" +nix-shell ./integration_tests/shell.nix --run "$cmd" diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go index 706c357b98..4337d96671 100644 --- a/versiondb/tsrocksdb/store.go +++ b/versiondb/tsrocksdb/store.go @@ -94,8 +94,10 @@ func (s Store) PutAtVersion(version int64, changeSet []types.StoreKVPair) error } func (s Store) GetAtVersionSlice(storeKey string, key []byte, version *int64) (*grocksdb.Slice, error) { + readOpts := newTSReadOptions(version) + defer readOpts.Destroy() value, ts, err := s.db.GetCFWithTS( - newTSReadOptions(version), + readOpts, s.cfHandle, prependStoreKey(storeKey, key), ) @@ -164,7 +166,9 @@ func (s Store) iteratorAtVersion(storeKey string, start, end []byte, version *in prefix := storePrefix(storeKey) start, end = iterateWithPrefix(prefix, start, end) - itr := s.db.NewIteratorCF(newTSReadOptions(version), s.cfHandle) + readOpts := newTSReadOptions(version) + defer readOpts.Destroy() + itr := s.db.NewIteratorCF(readOpts, s.cfHandle) return newRocksDBIterator(itr, prefix, start, end, reverse, s.skipVersionZero), nil } @@ -241,12 +245,10 @@ func (s Store) FixData(storeNames []string, dryRun bool) error { return err } } - - if !dryRun { - return s.Flush() + if dryRun { + return nil } - - return nil + return s.Flush() } // fixDataStore iterate the wrong data at version 0, parse the timestamp from the key and write it again. @@ -256,8 +258,11 @@ func (s Store) fixDataStore(storeName string, dryRun bool) error { return err } - batch := grocksdb.NewWriteBatch() - defer batch.Destroy() + var batch *grocksdb.WriteBatch + if !dryRun { + batch = grocksdb.NewWriteBatch() + defer batch.Destroy() + } prefix := storePrefix(storeName) readOpts := grocksdb.NewDefaultReadOptions() @@ -266,14 +271,14 @@ func (s Store) fixDataStore(storeName string, dryRun bool) error { realKey := cloneAppend(prefix, pair.Key) readOpts.SetTimestamp(pair.Timestamp) - oldValue, err := s.db.GetCF(readOpts, s.cfHandle, realKey) + oldValue, oldTimestamp, err := s.db.GetCFWithTS(readOpts, s.cfHandle, realKey) if err != nil { return err } - clean := bytes.Equal(oldValue.Data(), pair.Value) + clean := bytes.Equal(oldValue.Data(), pair.Value) && bytes.Equal(oldTimestamp.Data(), pair.Timestamp) oldValue.Free() - + oldTimestamp.Free() if clean { continue }