diff --git a/jsontests/Cargo.toml b/jsontests/Cargo.toml index 9441df3..e073d7d 100644 --- a/jsontests/Cargo.toml +++ b/jsontests/Cargo.toml @@ -2,7 +2,7 @@ name = "jsontests" version = "0.0.0" license = "Apache-2.0" -authors = ["Stewart Mackenzie ", "Wei Tang "] +authors = ["Stewart Mackenzie ", "Wei Tang ", "Mike Lubinets "] edition = "2018" [[bench]] @@ -20,10 +20,7 @@ env_logger = "0.5.11" sha3 = "0.6" ethereum-rlp = { version = "0.2", default-features = false } criterion = "0.2.5" - -[features] -default = [] -bench = [] +failure = "0.1.5" [target.'cfg(unix)'.dependencies] -gag = "0.1.10" +gag = "0.1.10" \ No newline at end of file diff --git a/jsontests/benches/performance.rs b/jsontests/benches/performance.rs index cb22405..c5ca1b6 100644 --- a/jsontests/benches/performance.rs +++ b/jsontests/benches/performance.rs @@ -12,10 +12,10 @@ use std::time::Duration; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmPerformance"] -#[test_with = "jsontests::util::run_test"] -#[bench_with = "jsontests::util::run_bench"] +#[test_with = "jsontests::vmtests::run_test"] +#[bench_with = "jsontests::vmtests::run_bench"] #[criterion_config = "criterion_cfg"] -struct _Performance; +struct Performance; pub fn criterion_cfg() -> Criterion { // Due to poor SputnikVM performance, there's no chance to get a lot of measurements @@ -25,3 +25,5 @@ pub fn criterion_cfg() -> Criterion { .measurement_time(Duration::from_secs(10)) .noise_threshold(0.07) } + +criterion_main!(Performance_bench_main); diff --git a/jsontests/jsontests-derive/Cargo.toml b/jsontests/jsontests-derive/Cargo.toml index f94becf..9ea8450 100644 --- a/jsontests/jsontests-derive/Cargo.toml +++ b/jsontests/jsontests-derive/Cargo.toml @@ -18,4 +18,5 @@ serde_json = "1.0" failure = "0.1.2" itertools = "0.7.8" criterion = "0.2.5" +rayon = "1.0.3" diff --git a/jsontests/jsontests-derive/src/attr.rs b/jsontests/jsontests-derive/src/attr.rs index 3b75ac2..0910192 100644 --- a/jsontests/jsontests-derive/src/attr.rs +++ b/jsontests/jsontests-derive/src/attr.rs @@ -5,6 +5,18 @@ use syn::Ident; use syn::Lit::Str; use syn::MetaItem; +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Runtime { + Static, + Dynamic, +} + +impl Default for Runtime { + fn default() -> Self { + Runtime::Dynamic + } +} + #[derive(Default)] pub struct Config { pub directory: String, @@ -12,6 +24,7 @@ pub struct Config { pub bench_with: Option, pub criterion_config: Option, pub patch: Option, + pub runtime: Runtime, pub skip: bool, pub should_panic: bool, } @@ -46,16 +59,13 @@ pub fn extract_attrs(ast: &syn::DeriveInput) -> Result { #[derive(JsonTests)]\n\ #[directory = \"../tests/testset\"]\n\ #[test_with = \"test::test_function\"]\n\ + #[runtime = \"static|dynamic\"] (Optional, default = dynamic)\n\ #[bench_wuth = \"test::bench_function\"] (Optional)\n\ #[patch = \"CustomTestPatch\" (Optional)\n\ #[skip] (Optional)\n\ #[should_panic] (Optional)\n\ struct TestSet;"; - if ast.attrs.len() < 2 || ast.attrs.len() > 6 { - return Err(failure::err_msg(ERROR_MSG)); - } - let config = ast .attrs .iter() @@ -81,6 +91,14 @@ pub fn extract_attrs(ast: &syn::DeriveInput) -> Result { patch: Some(ExternalRef::from(value.clone())), ..config }, + "runtime" => Config { + runtime: match value.as_ref() { + "static" => Runtime::Static, + "dynamic" => Runtime::Dynamic, + _ => panic!("{}", ERROR_MSG), + }, + ..config + }, _ => panic!("{}", ERROR_MSG), }, MetaItem::Word(ref ident) => match ident.as_ref() { diff --git a/jsontests/jsontests-derive/src/lib.rs b/jsontests/jsontests-derive/src/lib.rs index d60e9d7..13d9a77 100644 --- a/jsontests/jsontests-derive/src/lib.rs +++ b/jsontests/jsontests-derive/src/lib.rs @@ -14,16 +14,30 @@ use proc_macro::TokenStream; use syn::Ident; use self::{ - attr::{extract_attrs, Config}, + attr::{extract_attrs, Config, Runtime}, tests::read_tests_from_dir, util::*, }; +use crate::tests::{Test, TestAST, TestASTRunner}; +use std::ffi::OsStr; +use std::path::Path; #[proc_macro_derive( JsonTests, - attributes(directory, test_with, bench_with, criterion_config, skip, should_panic, patch) + attributes( + directory, + test_with, + bench_with, + criterion_config, + skip, + should_panic, + patch, + runtime + ) )] pub fn json_tests(input: TokenStream) -> TokenStream { + let timer = Timer::new("JsonTests proc-macro"); + // Construct a string representation of the type definition let s = input.to_string(); @@ -42,70 +56,93 @@ pub fn json_tests(input: TokenStream) -> TokenStream { fn impl_json_tests(ast: &syn::DeriveInput) -> Result { let config = extract_attrs(&ast)?; - let tests = read_tests_from_dir(&config.directory)?; + let tests = read_tests_from_dir(&config, &config.directory)?; let mut tokens = quote::Tokens::new(); - let mut bench_idents = Vec::new(); - - // split tests into groups by filepath - let tests = tests.group_by(|test| test.path.clone()); - - let dir_mod_name = open_directory_module(&config, &mut tokens); + let struct_ident = &ast.ident; // If behchmarking support is requested, import Criterion if config.bench_with.is_some() { tokens.append(quote! { - use criterion::Criterion; + use criterion::Criterion as _; }) } - for (filepath, tests) in &tests { - // If tests count in this file is 1, we don't need submodule - let tests = tests.collect::>(); - let need_file_submodule = tests.len() > 1; - let mut file_mod_name = None; - if need_file_submodule { - file_mod_name = Some(open_file_module(&filepath, &mut tokens)); - // If behchmarking support is requested, import Criterion - if config.bench_with.is_some() { - tokens.append(quote! { - use criterion::Criterion; - }) - } - } + let mut ast_runner = AstRunner { + config: &config, + tokens, + bench_idents: Vec::new(), + modules: Vec::new(), + }; - // Generate test function - for test in tests { - let name = sanitize_ident(&test.name); - let name_ident = Ident::from(name.as_ref()); - let data = json::to_string(&test.data)?; - - generate_test(&config, &name_ident, &data, &mut tokens); - generate_bench(&config, &name_ident, &data, &mut tokens).map(|mut ident| { - // prepend dir submodule - ident = Ident::from(format!("{}::{}", dir_mod_name, ident.as_ref())); - // prepend file submodule - if need_file_submodule { - ident = Ident::from(format!("{}::{}", file_mod_name.as_ref().unwrap(), ident.as_ref())); - } - bench_idents.push(ident); - }); - } + let traverse_timer = Timer::new("TestAST traverse"); + tests.traverse(&mut ast_runner); + drop(traverse_timer); - if need_file_submodule { - // Close file module - close_brace(&mut tokens) - } + let mut tokens = ast_runner.tokens; + + generate_criterion_macros(&config, &struct_ident, &ast_runner.bench_idents, &mut tokens); + + Ok(tokens) +} + +struct AstRunner<'a> { + config: &'a Config, + tokens: quote::Tokens, + modules: Vec, + bench_idents: Vec, +} + +impl AstRunner<'_> { + fn push_module(&mut self, name: String) { + let mod_name = sanitize_ident(&name); + self.modules.push(mod_name.clone()); + open_module(mod_name, &mut self.tokens); } - // Close directory module - close_brace(&mut tokens); + fn pop_module(&mut self) { + self.modules.pop(); + close_brace(&mut self.tokens) + } +} - generate_criterion_macros(&config, &bench_idents, &mut tokens); +impl TestASTRunner for AstRunner<'_> { + fn handle_test(&mut self, test: Test) { + let data = test.data.map(|d| json::to_string(&d).unwrap()); + let name = sanitize_ident(&test.name); + let name_ident = Ident::from(name.as_ref()); + generate_test(&self.config, &test.path, &name_ident, &data, &mut self.tokens); + generate_bench(&self.config, &test.path, &name_ident, &data, &mut self.tokens).map(|mut ident| { + // prepare sumbodule path + let modules_chain = self.modules.join("::"); + let bench_ident = format!("{}::{}", modules_chain, ident); + self.bench_idents.push(bench_ident.into()); + }); + } - Ok(tokens) + fn handle_open_module(&mut self, name: String, _nodes: &[TestAST]) { + self.push_module(name); + } + + fn handle_close_module(&mut self) { + self.pop_module() + } + + fn handle_open_test_file(&mut self, name: String, _nodes: &[TestAST]) { + self.push_module(name); + } + + fn handle_close_test_file(&mut self) { + self.pop_module() + } } -fn generate_test(config: &Config, test_name: &Ident, data: &str, tokens: &mut quote::Tokens) { +fn generate_test( + config: &Config, + path: impl AsRef, + test_name: &Ident, + data: &Option, + tokens: &mut quote::Tokens, +) { let test_func_path = &config.test_with.path; let test_func_name = &config.test_with.name; let test_name_str = test_name.as_ref(); @@ -116,17 +153,38 @@ fn generate_test(config: &Config, test_name: &Ident, data: &str, tokens: &mut qu tokens.append(quote! {#[should_panic]}); } - tokens.append(quote! { - fn #test_name() { - use #test_func_path; - use #patch_path; - let data = #data; - #test_func_name::<#patch_name>(#test_name_str, data); + match config.runtime { + Runtime::Static => { + let data = data.as_ref().unwrap(); + tokens.append(quote! { + pub(crate) fn #test_name() { + use #test_func_path; + use #patch_path; + let data = #data; + #test_func_name::<#patch_name>(#test_name_str, data); + } + }); + } + Runtime::Dynamic => { + let path = path.as_ref().to_str().unwrap(); + tokens.append(quote! { + pub(crate) fn #test_name() { + use #test_func_path; + use #patch_path; + jsontests::run_tests_from_file(#path, #test_func_name::<#patch_name>) + } + }); } - }); + } } -fn generate_bench(config: &Config, test_name: &Ident, data: &str, tokens: &mut quote::Tokens) -> Option { +fn generate_bench( + config: &Config, + path: impl AsRef, + test_name: &Ident, + data: &Option, + tokens: &mut quote::Tokens, +) -> Option { if config.bench_with.is_none() { return None; } @@ -140,19 +198,36 @@ fn generate_bench(config: &Config, test_name: &Ident, data: &str, tokens: &mut q let (patch_name, patch_path) = derive_patch(config); - tokens.append(quote! { - pub fn #bench_ident(c: &mut Criterion) { - use #bench_func_path; - use #patch_path; - let data = #data; - #bench_func_name::<#patch_name>(c, #bench_name, data); + match config.runtime { + Runtime::Static => { + let data = data.as_ref().unwrap(); + tokens.append(quote! { + pub(crate) fn #bench_ident(c: &mut criterion::Criterion) { + use #bench_func_path; + use #patch_path; + let data = #data; + #bench_func_name::<#patch_name>(c, #bench_name, data); + } + }); } - }); + Runtime::Dynamic => { + let path = path.as_ref().to_str().unwrap(); + tokens.append(quote! { + pub(crate) fn #bench_ident(c: &mut criterion::Criterion) { + use #bench_func_path; + use #patch_path; + jsontests::run_bench_from_file(c, #path, #bench_func_name::<#patch_name>) + } + }); + } + } Some(bench_ident) } -fn generate_criterion_macros(config: &Config, benches: &[Ident], tokens: &mut quote::Tokens) { +fn generate_criterion_macros(config: &Config, group_name: &Ident, benches: &[Ident], tokens: &mut quote::Tokens) { + let group_name = format!("{}_bench_main", group_name); + let group_name = Ident::from(sanitize_ident(&group_name)); // Generate criterion macros if config.bench_with.is_some() { let benches = benches.iter().map(AsRef::as_ref).join(" , "); @@ -160,13 +235,13 @@ fn generate_criterion_macros(config: &Config, benches: &[Ident], tokens: &mut qu .criterion_config .as_ref() .map(|cfg| cfg.path.clone()) - .unwrap_or_else(|| Ident::from("Criterion::default")); + .unwrap_or_else(|| Ident::from("criterion::Criterion::default")); let template = quote! { criterion_group! { - name = main; + name = #group_name; config = #config(); targets = TARGETS - }; + } }; tokens.append(template.as_ref().replace("TARGETS", &benches)); } diff --git a/jsontests/jsontests-derive/src/tests.rs b/jsontests/jsontests-derive/src/tests.rs index 46800b2..12c1f4e 100644 --- a/jsontests/jsontests-derive/src/tests.rs +++ b/jsontests/jsontests-derive/src/tests.rs @@ -1,50 +1,156 @@ use failure::Error; -use std::fs::{self, DirEntry, File}; +use std::collections::{HashMap, HashSet}; +use std::fs::{self, read, DirEntry, File, FileType}; use std::iter; -use std::path::Path; +use std::path::{Path, PathBuf}; +use crate::attr::{Config, Runtime}; use json::Value; use serde_json as json; pub struct Test { - pub path: String, + pub path: PathBuf, pub name: String, - pub data: Value, + pub data: Option, } -pub fn read_tests_from_dir>(dir_path: P) -> Result, Error> { - let dir = fs::read_dir(dir_path)?; +pub enum TestAST { + Module(String, Vec), + TestFile(String, Vec), + Test(Test), +} + +impl TestAST { + pub fn traverse(self, runner: &mut impl TestASTRunner) { + match self { + TestAST::Module(name, nodes) => { + runner.handle_open_module(name, &nodes); + nodes.into_iter().for_each(|n| Self::traverse(n, runner)); + runner.handle_close_module(); + } + TestAST::TestFile(name, nodes) => { + runner.handle_open_test_file(name, &nodes); + nodes.into_iter().for_each(|n| Self::traverse(n, runner)); + runner.handle_close_test_file(); + } + TestAST::Test(test) => { + runner.handle_test(test); + } + } + } +} + +pub trait TestASTRunner { + fn handle_test(&mut self, test: Test); + fn handle_open_module(&mut self, name: String, nodes: &[TestAST]); + fn handle_close_module(&mut self); + fn handle_open_test_file(&mut self, name: String, nodes: &[TestAST]); + fn handle_close_test_file(&mut self); +} - let iter = dir.flat_map(|file| match file { - Ok(file) => tests_iterator_from_direntry(&file).unwrap(), - Err(err) => panic!("failed to read dir: {}", err), - }); +use crate::util::Timer; +use rayon::prelude::*; +use std::io::Write; +use std::time::Instant; - Ok(iter) +pub fn read_tests_from_dir>(config: &Config, dir_path: P) -> Result { + let timer_msg = format!("reading tests from {}", dir_path.as_ref().display()); + let timer = Timer::new(&timer_msg); + read_module(config, dir_path) } -pub fn tests_iterator_from_direntry(file: &DirEntry) -> Result>, Error> { - let path = file.path().to_owned(); +fn read_dir_and_emit_nodes(config: &Config, path: impl AsRef) -> Result, Error> { + use std::convert::identity; + + // For every directory entry, running in parallel + fs::read_dir(path)? + .map(|entry| -> Result, Error> { + let entry = entry?; + let filetype = entry.file_type()?; + if filetype.is_dir() { + Ok(Some(read_module(config, entry.path())?)) + } else if filetype.is_file() { + Ok(read_tests_file(config, entry.path())?) + } else { + println!("ommitting symlink {}", entry.path().display()); + Ok(None) + } + }) + // Turn Result, E> into Option> + .map(Result::transpose) + // Remove all Nones (skipped symlinks) + .filter_map(identity) + // Collect Iter> into Result, E> + .collect() +} + +fn read_module(config: &Config, path: impl AsRef) -> Result { + assert!(path.as_ref().is_dir()); + + let name = path + .as_ref() + .components() + .last() + .ok_or_else(|| failure::err_msg("empty path"))? + .as_os_str() + .to_str() + .map(ToOwned::to_owned) + .ok_or_else(|| failure::err_msg("path is not a valid utf-8"))?; + + let children = read_dir_and_emit_nodes(config, path)?; + + Ok(TestAST::Module(name, children)) +} + +fn read_tests_file(config: &Config, path: impl AsRef) -> Result, Error> { + assert!(path.as_ref().is_file()); + + let path = path.as_ref().canonicalize()?.to_owned(); + + let name = path + .file_stem() + .ok_or_else(|| failure::err_msg("couldn't read file stem"))? + .to_str() + .map(ToOwned::to_owned) + .ok_or_else(|| failure::err_msg("path is not a valid utf-8"))?; // Skip non-json files if !path.extension().map(|e| e == "json").unwrap_or(false) { - return Ok(Box::new(iter::empty())); + return Ok(None); } - let file = File::open(&path)?; - let tests: Value = json::from_reader(file)?; + let tests: Vec<_>; + + match config.runtime { + // Static runtime includes the json test inside the test function body + // And can operate on per-test basis, unlike the dynamic runtime, which can only operate on files + Runtime::Static => { + let file = File::open(&path)?; + let jsontests: Value = json::from_reader(file)?; - // Move out the root object - let tests = match tests { - Value::Object(tests) => tests, - _ => panic!("expected a json object at the root of test file"), - }; + // Move out the root object + let jsontests = match jsontests { + Value::Object(t) => t, + _ => panic!("expected a json object at the root of test file"), + }; - let iter = tests.into_iter().map(move |(name, data)| Test { - path: path.to_str().unwrap().to_owned(), - name, - data, - }); + let iter = jsontests.into_iter().map(move |(name, data)| Test { + path: path.clone(), + name, + data: Some(data), + }); + + tests = iter.map(TestAST::Test).collect(); + } + Runtime::Dynamic => { + let test = Test { + path, + name: name.clone(), + data: None, + }; + tests = vec![TestAST::Test(test)] + } + } - Ok(Box::new(iter)) + Ok(Some(TestAST::TestFile(name, tests))) } diff --git a/jsontests/jsontests-derive/src/util.rs b/jsontests/jsontests-derive/src/util.rs index 8c7eaef..8634766 100644 --- a/jsontests/jsontests-derive/src/util.rs +++ b/jsontests/jsontests-derive/src/util.rs @@ -1,38 +1,34 @@ use quote; +use std::path::Path; use syn::Ident; use crate::attr::Config; +use std::ffi::OsStr; -pub fn open_directory_module(config: &Config, tokens: &mut quote::Tokens) -> String { - // get the leaf directory name - let dirname = config.directory.rsplit('/').next().unwrap(); - +pub fn open_directory_module(dirname: &str, tokens: &mut quote::Tokens) { // create identifier let dirname = sanitize_ident(dirname); let dirname_ident = Ident::from(dirname.as_ref()); open_module(dirname_ident, tokens); - - dirname } -pub fn open_file_module(filepath: &str, tokens: &mut quote::Tokens) -> String { +pub fn open_file_module(filepath: impl AsRef, tokens: &mut quote::Tokens) { + let filepath = filepath.as_ref(); // get file name without extension - let filename = filepath.rsplit('/').next().unwrap().split('.').next().unwrap(); + let filename = filepath.file_stem().and_then(OsStr::to_str).unwrap(); // create identifier let filename = sanitize_ident(filename); let filename_ident = Ident::from(filename.as_ref()); open_module(filename_ident, tokens); - - filename } pub fn open_module>(module_name: I, tokens: &mut quote::Tokens) { let module_name = module_name.into(); // append module opening tokens tokens.append(quote! { - mod #module_name + pub(crate) mod #module_name }); tokens.append("{"); } @@ -57,13 +53,61 @@ pub fn sanitize_ident(ident: &str) -> String { }; // replace special characters with _ - replace_chars(&ident, "!@#$%^&*-+=/<>;\'\"()`~", "_") + escape_as_underscore(&ident, "!@#$%^&*-+=/<>;\'\"()`~") } -pub fn replace_chars(s: &str, from: &str, to: &str) -> String { +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::time::Instant; + +thread_local! { + static UNDERSCORE_ENCODING_MAP: RefCell> = RefCell::new( + [('-', 1), ('_', 2)].iter().cloned().collect() + ); + static UNDERSCORE_ENCODING_MAX: Cell = Cell::new(0); +} + +fn escape_as_underscore(s: &str, from: &str) -> String { let mut initial = s.to_owned(); for c in from.chars() { - initial = initial.replace(c, to); + let replacement: String = UNDERSCORE_ENCODING_MAP.with(|map| { + let mut map = map.borrow_mut(); + let cnt = map.entry(c).or_insert_with(|| { + UNDERSCORE_ENCODING_MAX.with(|max| { + let cnt = max.get() + 1; + max.set(cnt); + cnt + }) + }); + std::iter::repeat('_').take(*cnt as usize).collect() + }); + initial = initial.replace(c, &replacement); } initial } + +pub struct Timer<'a> { + msg: &'a str, + start: Instant, +} + +impl<'a> Timer<'a> { + pub fn new(msg: &'a str) -> Self { + use std::io::Write; + eprintln!("[start] {}", msg); + Timer { + msg, + start: Instant::now(), + } + } +} + +impl<'a> Drop for Timer<'a> { + fn drop(&mut self) { + let elapsed = self.start.elapsed(); + let elapsed_secs = elapsed.as_secs(); + let elapsed_millis = elapsed.subsec_millis(); + let elapsed_secs_float = (elapsed_secs as f64) + (elapsed_millis as f64) / 1000.0; + eprintln!("[{:.3}] {} finished", elapsed_secs_float, self.msg); + } +} diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidAddress.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidAddress.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidAddress.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidAddress.json diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidBalance.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidBalance.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidBalance.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidBalance.json diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidCode.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidCode.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidCode.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidCode.json diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidGas.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidGas.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidGas.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidGas.json diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidLogs.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidLogs.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidLogs.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidLogs.json diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidNonce.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidNonce.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidNonce.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidNonce.json diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidOutValue.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidOutValue.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidOutValue.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidOutValue.json diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidStorage.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidStorage.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidStorage.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidStorage.json diff --git a/jsontests/res/files/HarnessCorrectnessTests/invalidStorageAddress.json b/jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidStorageAddress.json similarity index 100% rename from jsontests/res/files/HarnessCorrectnessTests/invalidStorageAddress.json rename to jsontests/res/files/VMTestsHarnessCorrectnessTests/invalidStorageAddress.json diff --git a/jsontests/src/blockchaintests/mod.rs b/jsontests/src/blockchaintests/mod.rs new file mode 100644 index 0000000..1d82fb3 --- /dev/null +++ b/jsontests/src/blockchaintests/mod.rs @@ -0,0 +1,3 @@ +mod util; + +pub use util::run_test; diff --git a/jsontests/src/blockchaintests/util.rs b/jsontests/src/blockchaintests/util.rs new file mode 100644 index 0000000..3dbae24 --- /dev/null +++ b/jsontests/src/blockchaintests/util.rs @@ -0,0 +1,7 @@ +use evm::Patch; +use serde_json as json; +use serde_json::Value; + +pub fn run_test(name: &str, test: &str) { + unimplemented!() +} diff --git a/jsontests/src/lib.rs b/jsontests/src/lib.rs index 1aad9fc..49c0754 100644 --- a/jsontests/src/lib.rs +++ b/jsontests/src/lib.rs @@ -1,358 +1,48 @@ -#![cfg_attr(feature = "bench", feature(test))] -extern crate bigint; -extern crate criterion; -extern crate env_logger; -extern crate evm; -extern crate hexutil; -extern crate rlp; -extern crate serde_json; -extern crate sha3; - -mod blockchain; -pub mod util; - -pub use self::blockchain::{create_block, create_context, JSONBlock}; - -use bigint::{Address, Gas, H256, M256, U256}; -use evm::errors::RequireError; -use evm::{Context, Patch, SeqContextVM, VMStatus, VM}; -use hexutil::*; +use evm::Patch; use serde_json::Value; -use std::ops::Deref; -use std::str::FromStr; -use std::sync::{Arc, Mutex}; +use std::fs::File; +use std::path::Path; -pub fn fire_with_block(machine: &mut SeqContextVM

, block: &JSONBlock) { - loop { - match machine.fire() { - Err(RequireError::Account(address)) => { - let account = block.request_account(address); - machine.commit_account(account).unwrap(); - } - Err(RequireError::AccountCode(address)) => { - let account = block.request_account_code(address); - machine.commit_account(account).unwrap(); - } - Err(RequireError::AccountStorage(address, index)) => { - let account = block.request_account_storage(address, index); - machine.commit_account(account).unwrap(); - } - Err(RequireError::Blockhash(number)) => { - // The test JSON file doesn't expose any block - // information. So those numbers are crafted by hand. - let hash1 = - H256::from_str("0xc89efdaa54c0f20c7adf612882df0950f5a951637e0307cdcb4c672f298b8bc6").unwrap(); - let hash2 = - H256::from_str("0xad7c5bef027816a800da1736444fb58a807ef4c9603b7848673f7e3a68eb14a5").unwrap(); - let hash256 = - H256::from_str("0x6ca54da2c4784ea43fd88b3402de07ae4bced597cbb19f323b7595857a6720ae").unwrap(); +pub mod blockchaintests; +pub mod vmtests; - let hash = if number == U256::from(1u64) { - hash1 - } else if number == U256::from(2u64) { - hash2 - } else if number == U256::from(256u64) { - hash256 - } else { - panic!(); - }; +fn load_tests(path: impl AsRef) -> impl Iterator { + let tests = || -> Result<_, failure::Error> { + let file = File::open(&path)?; + Ok(serde_json::from_reader(file)?) + }(); - machine.commit_blockhash(number, hash).unwrap(); - } - Ok(()) => return, - } - } -} + println!("{}", std::env::current_dir().unwrap().display()); -pub fn apply_to_block(machine: &SeqContextVM

, block: &mut JSONBlock) { - for account in machine.accounts() { - let account = (*account).clone(); - block.apply_account(account); - } - for log in machine.logs() { - let log = (*log).clone(); - block.apply_log(log); - } -} - -pub fn create_machine<'a, P: Patch>(patch: &'a P, v: &Value, block: &JSONBlock) -> SeqContextVM<'a, P> { - let transaction = create_context(v); + // Move out the root object + let tests = match tests { + Ok(Value::Object(tests)) => tests, + Ok(_) => panic!("expected a json object at the root of test file"), + Err(e) => panic!("failed to open test file at {}: {}", path.as_ref().display(), e), + }; - SeqContextVM::new(patch, transaction, block.block_header()) + tests.into_iter() } -// TODO: consider refactoring -#[cfg_attr(feature = "cargo-clippy", allow(cyclomatic_complexity))] -#[cfg_attr(feature = "cargo-clippy", allow(collapsible_if))] -pub fn test_machine( - v: &Value, - machine: &SeqContextVM

, - block: &JSONBlock, - history: &[Context], - debug: bool, -) -> bool { - let callcreates = &v["callcreates"]; - - if callcreates.as_array().is_some() { - for (i, callcreate) in callcreates.as_array().unwrap().into_iter().enumerate() { - let data = read_hex(callcreate["data"].as_str().unwrap()).unwrap(); - let destination = { - let destination = callcreate["destination"].as_str().unwrap(); - if destination == "" { - None - } else { - Some(Address::from_str(destination).unwrap()) - } - }; - let gas_limit = Gas::from(read_u256(callcreate["gasLimit"].as_str().unwrap())); - let value = read_u256(callcreate["value"].as_str().unwrap()); - - if i >= history.len() { - if debug { - println!(); - println!("Transaction check failed, expected more than {} items.", i); - } - return false; - } - let transaction = &history[i]; - if destination.is_some() { - if transaction.address != destination.unwrap() { - if debug { - println!(); - println!( - "Transaction address mismatch. 0x{:x} != 0x{:x}.", - transaction.address, - destination.unwrap() - ); - } - return false; - } - } - if transaction.gas_limit != gas_limit - || transaction.value != value - || if destination.is_some() { - transaction.data.deref() != &data - } else { - transaction.code.deref() != &data - } - { - if debug { - println!(); - println!("Transaction mismatch. gas limit 0x{:x} =?= 0x{:x}, value 0x{:x} =?= 0x{:x}, data {:?} =?= {:?}", transaction.gas_limit, gas_limit, transaction.value, value, transaction.data, data); - } - return false; - } - } - } - - let out = v["out"].as_str(); - let gas = v["gas"].as_str(); - - if let Some(out) = out { - let out = read_hex(out).unwrap(); - let out_ref: &[u8] = out.as_ref(); - if machine.out() != out_ref { - if debug { - println!(); - println!("Return value check failed. {:?} != {:?}", machine.out(), out_ref); - } - - return false; - } - } - - if let Some(gas) = gas { - let gas = Gas::from(read_u256(gas)); - if machine.available_gas() != gas { - if debug { - println!(); - println!( - "Gas check failed, VM returned: 0x{:x}, expected: 0x{:x}", - machine.available_gas(), - gas - ); - } - - return false; - } - } - - let post_addresses = &v["post"]; - - for (address, data) in post_addresses.as_object().unwrap() { - let address = Address::from_str(address.as_str()).unwrap(); - let balance = read_u256(data["balance"].as_str().unwrap()); - let nonce = read_u256(data["nonce"].as_str().unwrap()); - let code = read_hex(data["code"].as_str().unwrap()).unwrap(); - let code_ref: &[u8] = code.as_ref(); - - if code_ref != block.account_code(address) { - if debug { - println!(); - println!("Account code check failed for address 0x{:x}.", address); - println!("Expected: {:x?}", block.account_code(address)); - println!("Actual: {:x?}", code_ref); - } - - return false; - } - - if balance != block.balance(address) { - if debug { - println!(); - println!("Balance check failed for address 0x{:x}.", address); - println!("Expected: 0x{:x}", balance); - println!("Actual: 0x{:x}", block.balance(address)); - } - - return false; - } - - if nonce != block.nonce(address) { - if debug { - println!(); - println!("Nonce check failed for address 0x{:x}.", address); - println!("Expected: 0x{:x}", nonce); - println!("Actual: 0x{:x}", block.nonce(address)); - } - - return false; - } - - let storage = data["storage"].as_object().unwrap(); - for (index, value) in storage { - let index = read_u256(index.as_str()); - let value = M256::from_str(value.as_str().unwrap()).unwrap(); - if value != block.account_storage(address, index) { - if debug { - println!(); - println!( - "Storage check failed for address 0x{:x} in storage index 0x{:x}", - address, index - ); - println!("Expected: 0x{:x}", value); - println!("Actual: 0x{:x}", block.account_storage(address, index)); - } - return false; - } - } - } - - let expect = &v["expect"]; - - if expect.as_object().is_some() { - for (address, data) in expect.as_object().unwrap() { - let address = Address::from_str(address.as_str()).unwrap(); - - let storage = data["storage"].as_object().unwrap(); - for (index, value) in storage { - let index = read_u256(index.as_str()); - let value = M256::from_str(value.as_str().unwrap()).unwrap(); - if value != block.account_storage(address, index) { - if debug { - println!(); - println!( - "Storage check (expect) failed for address 0x{:x} in storage index 0x{:x}", - address, index - ); - println!("Expected: 0x{:x}", value); - println!("Actual: 0x{:x}", block.account_storage(address, index)); - } - return false; - } - } - } - } - - let logs_hash = v["logs"].as_str().map(read_u256); - - if logs_hash.is_some() { - let logs_hash = logs_hash.unwrap(); - let vm_logs_hash = block.logs_rlp_hash(); - if logs_hash != vm_logs_hash { - if debug { - println!(); - println!("Logs check failed (hashes mismatch)"); - println!("Expected: 0x{:x}", logs_hash); - println!("Actual: 0x{:x}", vm_logs_hash); - } - return false; - } - } - - true -} - -fn is_ok(status: &VMStatus) -> bool { - match *status { - VMStatus::ExitedOk => true, - _ => false, - } -} - -pub fn test_transaction(_name: &str, patch: P, v: &Value, debug: bool) -> Result { - let _ = env_logger::try_init(); - - let mut block = create_block(v); - let history: Arc>> = Arc::new(Mutex::new(Vec::new())); - let history_closure = history.clone(); - let mut machine = create_machine::

(&patch, v, &block); - machine.add_context_history_hook(move |context| { - history_closure.lock().unwrap().push(context.clone()); - }); - fire_with_block(&mut machine, &block); - apply_to_block(&machine, &mut block); - - if debug { - println!("status: {:?}", machine.status()); - } - let out = v["out"].as_str(); - - if out.is_some() { - if is_ok(&machine.status()) { - if test_machine(v, &machine, &block, &history.lock().unwrap(), debug) { - Ok(true) - } else { - Ok(false) - } - } else { - Err(machine.status()) - } - } else if !is_ok(&machine.status()) { - Ok(true) - } else { - Ok(false) +pub fn run_tests_from_file(path: impl AsRef, test_fn: F) +where + F: Fn(&str, &str), +{ + let tests = load_tests(path); + for (name, test) in tests { + let test_data = serde_json::to_string(&test).unwrap(); + test_fn(&name, &test_data) } } - use criterion::Criterion; -pub fn bench_transaction(_name: &str, patch: P, v: Value, c: &mut Criterion) { - c.bench_function(_name, move |b| { - b.iter_with_large_setup( - || { - let block = create_block(&v); - let history: Arc>> = Arc::new(Mutex::new(Vec::new())); - let history_closure = history.clone(); - let mut machine = create_machine(&patch, &v, &block); - machine.add_context_history_hook(move |context| { - history_closure.lock().unwrap().push(context.clone()); - }); - (machine, block) - }, - |(mut machine, block)| { - fire_with_block(&mut machine, &block); - }, - ) - }); -} - -/// Read U256 number exactly the way go big.Int parses strings -/// except for base 2 and 8 which are not used in tests -pub fn read_u256(number: &str) -> U256 { - if number.starts_with("0x") { - U256::from_str(number).unwrap() - } else { - U256::from_dec_str(number).unwrap() +pub fn run_bench_from_file(c: &mut Criterion, path: impl AsRef, bench_fn: F) +where + F: Fn(&mut Criterion, &str, &str) + 'static, +{ + let tests = load_tests(path); + for (name, test) in tests { + let test_data = serde_json::to_string(&test).unwrap(); + bench_fn(c, &name, &test_data) } } diff --git a/jsontests/src/blockchain.rs b/jsontests/src/vmtests/blockchain.rs similarity index 99% rename from jsontests/src/blockchain.rs rename to jsontests/src/vmtests/blockchain.rs index 11e5e84..4da9442 100644 --- a/jsontests/src/blockchain.rs +++ b/jsontests/src/vmtests/blockchain.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; use std::rc::Rc; use std::str::FromStr; -use crate::read_u256; +use super::read_u256; pub struct JSONBlock { codes: HashMap>, diff --git a/jsontests/src/vmtests/mod.rs b/jsontests/src/vmtests/mod.rs new file mode 100644 index 0000000..b7ca879 --- /dev/null +++ b/jsontests/src/vmtests/mod.rs @@ -0,0 +1,349 @@ +mod blockchain; +mod util; + +pub use self::blockchain::{create_block, create_context, JSONBlock}; +pub use self::util::{run_bench, run_test}; + +use bigint::{Address, Gas, H256, M256, U256}; +use evm::errors::RequireError; +use evm::{Context, Patch, SeqContextVM, VMStatus, VM}; +use hexutil::*; +use serde_json::Value; +use std::ops::Deref; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; + +pub fn fire_with_block(machine: &mut SeqContextVM

, block: &JSONBlock) { + loop { + match machine.fire() { + Err(RequireError::Account(address)) => { + let account = block.request_account(address); + machine.commit_account(account).unwrap(); + } + Err(RequireError::AccountCode(address)) => { + let account = block.request_account_code(address); + machine.commit_account(account).unwrap(); + } + Err(RequireError::AccountStorage(address, index)) => { + let account = block.request_account_storage(address, index); + machine.commit_account(account).unwrap(); + } + Err(RequireError::Blockhash(number)) => { + // The test JSON file doesn't expose any block + // information. So those numbers are crafted by hand. + let hash1 = + H256::from_str("0xc89efdaa54c0f20c7adf612882df0950f5a951637e0307cdcb4c672f298b8bc6").unwrap(); + let hash2 = + H256::from_str("0xad7c5bef027816a800da1736444fb58a807ef4c9603b7848673f7e3a68eb14a5").unwrap(); + let hash256 = + H256::from_str("0x6ca54da2c4784ea43fd88b3402de07ae4bced597cbb19f323b7595857a6720ae").unwrap(); + + let hash = if number == U256::from(1u64) { + hash1 + } else if number == U256::from(2u64) { + hash2 + } else if number == U256::from(256u64) { + hash256 + } else { + panic!(); + }; + + machine.commit_blockhash(number, hash).unwrap(); + } + Ok(()) => return, + } + } +} + +pub fn apply_to_block(machine: &SeqContextVM

, block: &mut JSONBlock) { + for account in machine.accounts() { + let account = (*account).clone(); + block.apply_account(account); + } + for log in machine.logs() { + let log = (*log).clone(); + block.apply_log(log); + } +} + +pub fn create_machine<'a, P: Patch>(patch: &'a P, v: &Value, block: &JSONBlock) -> SeqContextVM<'a, P> { + let transaction = create_context(v); + + SeqContextVM::new(patch, transaction, block.block_header()) +} + +// TODO: consider refactoring +#[cfg_attr(feature = "cargo-clippy", allow(cyclomatic_complexity))] +#[cfg_attr(feature = "cargo-clippy", allow(collapsible_if))] +pub fn test_machine( + v: &Value, + machine: &SeqContextVM

, + block: &JSONBlock, + history: &[Context], + debug: bool, +) -> bool { + let callcreates = &v["callcreates"]; + + if callcreates.as_array().is_some() { + for (i, callcreate) in callcreates.as_array().unwrap().into_iter().enumerate() { + let data = read_hex(callcreate["data"].as_str().unwrap()).unwrap(); + let destination = { + let destination = callcreate["destination"].as_str().unwrap(); + if destination == "" { + None + } else { + Some(Address::from_str(destination).unwrap()) + } + }; + let gas_limit = Gas::from(read_u256(callcreate["gasLimit"].as_str().unwrap())); + let value = read_u256(callcreate["value"].as_str().unwrap()); + + if i >= history.len() { + if debug { + println!(); + println!("Transaction check failed, expected more than {} items.", i); + } + return false; + } + let transaction = &history[i]; + if destination.is_some() { + if transaction.address != destination.unwrap() { + if debug { + println!(); + println!( + "Transaction address mismatch. 0x{:x} != 0x{:x}.", + transaction.address, + destination.unwrap() + ); + } + return false; + } + } + if transaction.gas_limit != gas_limit + || transaction.value != value + || if destination.is_some() { + transaction.data.deref() != &data + } else { + transaction.code.deref() != &data + } + { + if debug { + println!(); + println!("Transaction mismatch. gas limit 0x{:x} =?= 0x{:x}, value 0x{:x} =?= 0x{:x}, data {:?} =?= {:?}", transaction.gas_limit, gas_limit, transaction.value, value, transaction.data, data); + } + return false; + } + } + } + + let out = v["out"].as_str(); + let gas = v["gas"].as_str(); + + if let Some(out) = out { + let out = read_hex(out).unwrap(); + let out_ref: &[u8] = out.as_ref(); + if machine.out() != out_ref { + if debug { + println!(); + println!("Return value check failed. {:?} != {:?}", machine.out(), out_ref); + } + + return false; + } + } + + if let Some(gas) = gas { + let gas = Gas::from(read_u256(gas)); + if machine.available_gas() != gas { + if debug { + println!(); + println!( + "Gas check failed, VM returned: 0x{:x}, expected: 0x{:x}", + machine.available_gas(), + gas + ); + } + + return false; + } + } + + let post_addresses = &v["post"]; + + for (address, data) in post_addresses.as_object().unwrap() { + let address = Address::from_str(address.as_str()).unwrap(); + let balance = read_u256(data["balance"].as_str().unwrap()); + let nonce = read_u256(data["nonce"].as_str().unwrap()); + let code = read_hex(data["code"].as_str().unwrap()).unwrap(); + let code_ref: &[u8] = code.as_ref(); + + if code_ref != block.account_code(address) { + if debug { + println!(); + println!("Account code check failed for address 0x{:x}.", address); + println!("Expected: {:x?}", block.account_code(address)); + println!("Actual: {:x?}", code_ref); + } + + return false; + } + + if balance != block.balance(address) { + if debug { + println!(); + println!("Balance check failed for address 0x{:x}.", address); + println!("Expected: 0x{:x}", balance); + println!("Actual: 0x{:x}", block.balance(address)); + } + + return false; + } + + if nonce != block.nonce(address) { + if debug { + println!(); + println!("Nonce check failed for address 0x{:x}.", address); + println!("Expected: 0x{:x}", nonce); + println!("Actual: 0x{:x}", block.nonce(address)); + } + + return false; + } + + let storage = data["storage"].as_object().unwrap(); + for (index, value) in storage { + let index = read_u256(index.as_str()); + let value = M256::from_str(value.as_str().unwrap()).unwrap(); + if value != block.account_storage(address, index) { + if debug { + println!(); + println!( + "Storage check failed for address 0x{:x} in storage index 0x{:x}", + address, index + ); + println!("Expected: 0x{:x}", value); + println!("Actual: 0x{:x}", block.account_storage(address, index)); + } + return false; + } + } + } + + let expect = &v["expect"]; + + if expect.as_object().is_some() { + for (address, data) in expect.as_object().unwrap() { + let address = Address::from_str(address.as_str()).unwrap(); + + let storage = data["storage"].as_object().unwrap(); + for (index, value) in storage { + let index = read_u256(index.as_str()); + let value = M256::from_str(value.as_str().unwrap()).unwrap(); + if value != block.account_storage(address, index) { + if debug { + println!(); + println!( + "Storage check (expect) failed for address 0x{:x} in storage index 0x{:x}", + address, index + ); + println!("Expected: 0x{:x}", value); + println!("Actual: 0x{:x}", block.account_storage(address, index)); + } + return false; + } + } + } + } + + let logs_hash = v["logs"].as_str().map(read_u256); + + if logs_hash.is_some() { + let logs_hash = logs_hash.unwrap(); + let vm_logs_hash = block.logs_rlp_hash(); + if logs_hash != vm_logs_hash { + if debug { + println!(); + println!("Logs check failed (hashes mismatch)"); + println!("Expected: 0x{:x}", logs_hash); + println!("Actual: 0x{:x}", vm_logs_hash); + } + return false; + } + } + + true +} + +fn is_ok(status: &VMStatus) -> bool { + match *status { + VMStatus::ExitedOk => true, + _ => false, + } +} + +pub fn test_transaction(_name: &str, patch: P, v: &Value, debug: bool) -> Result { + let _ = env_logger::try_init(); + + let mut block = create_block(v); + let history: Arc>> = Arc::new(Mutex::new(Vec::new())); + let history_closure = history.clone(); + let mut machine = create_machine::

(&patch, v, &block); + machine.add_context_history_hook(move |context| { + history_closure.lock().unwrap().push(context.clone()); + }); + fire_with_block(&mut machine, &block); + apply_to_block(&machine, &mut block); + + if debug { + println!("status: {:?}", machine.status()); + } + let out = v["out"].as_str(); + + if out.is_some() { + if is_ok(&machine.status()) { + if test_machine(v, &machine, &block, &history.lock().unwrap(), debug) { + Ok(true) + } else { + Ok(false) + } + } else { + Err(machine.status()) + } + } else if !is_ok(&machine.status()) { + Ok(true) + } else { + Ok(false) + } +} + +use criterion::Criterion; + +pub fn bench_transaction(_name: &str, patch: P, v: Value, c: &mut Criterion) { + c.bench_function(_name, move |b| { + b.iter_with_large_setup( + || { + let block = create_block(&v); + let history: Arc>> = Arc::new(Mutex::new(Vec::new())); + let history_closure = history.clone(); + let mut machine = create_machine(&patch, &v, &block); + machine.add_context_history_hook(move |context| { + history_closure.lock().unwrap().push(context.clone()); + }); + (machine, block) + }, + |(mut machine, block)| { + fire_with_block(&mut machine, &block); + }, + ) + }); +} + +/// Read U256 number exactly the way go big.Int parses strings +/// except for base 2 and 8 which are not used in tests +pub fn read_u256(number: &str) -> U256 { + if number.starts_with("0x") { + U256::from_str(number).unwrap() + } else { + U256::from_dec_str(number).unwrap() + } +} diff --git a/jsontests/src/util.rs b/jsontests/src/vmtests/util.rs similarity index 84% rename from jsontests/src/util.rs rename to jsontests/src/vmtests/util.rs index 2a79a60..59ff4cb 100644 --- a/jsontests/src/util.rs +++ b/jsontests/src/vmtests/util.rs @@ -2,7 +2,7 @@ use evm::Patch; use serde_json as json; use serde_json::Value; -use crate::{bench_transaction, test_transaction}; +use super::{bench_transaction, test_transaction}; pub fn run_test(name: &str, test: &str) { let test: Value = json::from_str(test).unwrap(); @@ -11,7 +11,7 @@ pub fn run_test(name: &str, test: &str) { use criterion::Criterion; -pub fn run_bench(c: &mut Criterion, name: &'static str, test: &str) { +pub fn run_bench(c: &mut Criterion, name: &str, test: &str) { let test: Value = json::from_str(test).unwrap(); bench_transaction(name, P::default(), test, c); } diff --git a/jsontests/tests/blockchaintests.rs b/jsontests/tests/blockchaintests.rs new file mode 100644 index 0000000..2543a9f --- /dev/null +++ b/jsontests/tests/blockchaintests.rs @@ -0,0 +1,86 @@ +#![allow(non_snake_case)] +#![allow(unused)] + +#[macro_use] +extern crate jsontests_derive; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/GeneralStateTests"] +#[test_with = "jsontests::blockchaintests::run_test"] +#[runtime = "dynamic"] +struct GeneralStateTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/TransitionTests"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct TransitionTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcBlockGasLimitTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct BlockGasLimitTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcExploitTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct ExploitTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcForgedTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct ForgedTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcForkStressTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct ForkStressTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcGasPricerTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct GasPricerTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcInvalidHeaderTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct InvalidHeaderTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcMultiChainTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct MultiChainTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcRandomBlockhashTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct RandomBlockhashTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcStateTests"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct StateTests; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcTotalDifficultyTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct TotalDifficultyTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcUncleHeaderValidity"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct UncleHeaderValidity; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcUncleTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct UncleTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcValidBlockTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct ValidBlockTest; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/BlockchainTests/bcWalletTest"] +#[test_with = "jsontests::blockchaintests::run_test"] +struct WalletTest; diff --git a/jsontests/tests/eiptests.rs b/jsontests/tests/eiptests.rs deleted file mode 100644 index ac25bce..0000000 --- a/jsontests/tests/eiptests.rs +++ /dev/null @@ -1,79 +0,0 @@ -#![cfg_attr(feature = "bench", feature(test))] -#![allow(non_snake_case)] -#![allow(unused)] - -#[macro_use] -extern crate jsontests_derive; - -use bigint::{Address, Gas}; -use evm::{EmbeddedAccountPatch, Patch, Precompiled, EMBEDDED_PRECOMPILEDS}; - -// Shifting opcodes tests -#[derive(JsonTests)] -#[directory = "jsontests/res/files/eth/VMTests/vmEIP215"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] -struct EIP215; - -// EXTCODEHASH tests -#[derive(JsonTests)] -#[directory = "jsontests/res/files/eth/VMTests/vmEIP1052"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] -struct EIP1052; - -// CREATE2 tests -#[derive(JsonTests)] -#[directory = "jsontests/res/files/eth/VMTests/vmEIP1014"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] -struct EIP1014; - -// Gas metering changes tests -#[derive(JsonTests)] -#[directory = "jsontests/res/files/eth/VMTests/vmEIP1283"] -#[test_with = "jsontests::util::run_test"] -#[patch = "crate::EIP1283Patch"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] -struct EIP1283; - -#[derive(Copy, Clone, Default)] -struct EIP1283Patch(pub EmbeddedAccountPatch); - -#[rustfmt::skip] -impl Patch for EIP1283Patch { - type Account = EmbeddedAccountPatch; - - fn account_patch(&self) -> &Self::Account { &self.0 } - fn code_deposit_limit(&self) -> Option { None } - fn callstack_limit(&self) -> usize { 2 } - fn gas_extcode(&self) -> Gas { Gas::from(20usize) } - fn gas_balance(&self) -> Gas { Gas::from(20usize) } - fn gas_sload(&self) -> Gas { Gas::from(50usize) } - fn gas_suicide(&self) -> Gas { Gas::from(0usize) } - fn gas_suicide_new_account(&self) -> Gas { Gas::from(0usize) } - fn gas_call(&self) -> Gas { Gas::from(40usize) } - fn gas_expbyte(&self) -> Gas { Gas::from(10usize) } - fn gas_transaction_create(&self) -> Gas { Gas::from(0usize) } - fn force_code_deposit(&self) -> bool { true } - fn has_delegate_call(&self) -> bool { true } - fn has_static_call(&self) -> bool { true } - fn has_revert(&self) -> bool { true } - fn has_return_data(&self) -> bool { true } - fn has_bitwise_shift(&self) -> bool { true } - fn has_create2(&self) -> bool { true } - fn has_extcodehash(&self) -> bool { true } - fn has_reduced_sstore_gas_metering(&self) -> bool { true } - fn err_on_call_with_more_gas(&self) -> bool { true } - fn call_create_l64_after_gas(&self) -> bool { false } - fn memory_limit(&self) -> usize { usize::max_value() } - fn is_precompiled_contract_enabled(&self, address: &Address) -> bool { - match address.low_u64() { - 0x1 | 0x2 | 0x3 | 0x4 => true, - _ => false, - } - } - fn precompileds(&self) -> &'static [(Address, Option<&'static [u8]>, &'static Precompiled)] { - &EMBEDDED_PRECOMPILEDS - } -} diff --git a/jsontests/tests/harness.rs b/jsontests/tests/harness.rs index 054a549..d4f0397 100644 --- a/jsontests/tests/harness.rs +++ b/jsontests/tests/harness.rs @@ -5,7 +5,13 @@ extern crate jsontests_derive; #[derive(JsonTests)] -#[directory = "jsontests/res/files/HarnessCorrectnessTests"] -#[test_with = "jsontests::util::run_test"] +#[directory = "jsontests/res/files/VMTestsHarnessCorrectnessTests"] +#[test_with = "jsontests::vmtests::run_test"] #[should_panic] -struct HarnessCorrectness; +struct VMTestsHarnessCorrectness; + +#[derive(JsonTests)] +#[directory = "jsontests/res/files/BlockchainTestsHarnessCorrectnessTests"] +#[test_with = "jsontests::vmtests::run_test"] +#[should_panic] +struct BlockchainTestsHarnessCorrectionTests; diff --git a/jsontests/tests/inputlimits.rs b/jsontests/tests/inputlimits.rs index f8bb6be..1a932d4 100644 --- a/jsontests/tests/inputlimits.rs +++ b/jsontests/tests/inputlimits.rs @@ -1,7 +1,7 @@ #![allow(non_snake_case)] use evm::VMTestPatch; -use jsontests::test_transaction; +use jsontests::vmtests::test_transaction; use serde_json::Value; // Log format is broken for input limits tests diff --git a/jsontests/tests/vmtests.rs b/jsontests/tests/vmtests.rs index ed6fcb5..b198a12 100644 --- a/jsontests/tests/vmtests.rs +++ b/jsontests/tests/vmtests.rs @@ -1,75 +1,129 @@ -#![cfg_attr(feature = "bench", feature(test))] #![allow(non_snake_case)] #![allow(unused)] #[macro_use] extern crate jsontests_derive; -#[cfg(feature = "bench")] -extern crate test; - #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmArithmeticTest"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct Arithmetic; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmBitwiseLogicOperation"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct BitwiseLogicOperation; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmBlockInfoTest"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct BlockInfo; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmEnvironmentalInfo"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct VmInverontemtalInfo; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmIOandFlowOperations"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct VmIOandFlowOperations; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmLogTest"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct Log; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmPushDupSwapTest"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct PushDupSwap; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmRandomTest"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct Random; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmSha3Test"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct Sha3; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmSystemOperations"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct SystemOperations; #[derive(JsonTests)] #[directory = "jsontests/res/files/eth/VMTests/vmTests"] -#[test_with = "jsontests::util::run_test"] -#[cfg_attr(feature = "bench", bench_with = "jsontests::util::run_bench")] +#[test_with = "jsontests::vmtests::run_test"] struct VM; + +use bigint::{Address, Gas}; +use evm::{EmbeddedAccountPatch, Patch, Precompiled, EMBEDDED_PRECOMPILEDS}; + +// Shifting opcodes tests +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/VMTests/vmEIP215"] +#[test_with = "jsontests::vmtests::run_test"] +struct EIP215; + +// EXTCODEHASH tests +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/VMTests/vmEIP1052"] +#[test_with = "jsontests::vmtests::run_test"] +struct EIP1052; + +// CREATE2 tests +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/VMTests/vmEIP1014"] +#[test_with = "jsontests::vmtests::run_test"] +struct EIP1014; + +// Gas metering changes tests +#[derive(JsonTests)] +#[directory = "jsontests/res/files/eth/VMTests/vmEIP1283"] +#[test_with = "jsontests::vmtests::run_test"] +#[patch = "crate::EIP1283Patch"] +struct EIP1283; + +#[derive(Copy, Clone, Default)] +struct EIP1283Patch(pub EmbeddedAccountPatch); + +#[rustfmt::skip] +impl Patch for EIP1283Patch { + type Account = EmbeddedAccountPatch; + + fn account_patch(&self) -> &Self::Account { &self.0 } + fn code_deposit_limit(&self) -> Option { None } + fn callstack_limit(&self) -> usize { 2 } + fn gas_extcode(&self) -> Gas { Gas::from(20usize) } + fn gas_balance(&self) -> Gas { Gas::from(20usize) } + fn gas_sload(&self) -> Gas { Gas::from(50usize) } + fn gas_suicide(&self) -> Gas { Gas::from(0usize) } + fn gas_suicide_new_account(&self) -> Gas { Gas::from(0usize) } + fn gas_call(&self) -> Gas { Gas::from(40usize) } + fn gas_expbyte(&self) -> Gas { Gas::from(10usize) } + fn gas_transaction_create(&self) -> Gas { Gas::from(0usize) } + fn force_code_deposit(&self) -> bool { true } + fn has_delegate_call(&self) -> bool { true } + fn has_static_call(&self) -> bool { true } + fn has_revert(&self) -> bool { true } + fn has_return_data(&self) -> bool { true } + fn has_bitwise_shift(&self) -> bool { true } + fn has_create2(&self) -> bool { true } + fn has_extcodehash(&self) -> bool { true } + fn has_reduced_sstore_gas_metering(&self) -> bool { true } + fn err_on_call_with_more_gas(&self) -> bool { true } + fn call_create_l64_after_gas(&self) -> bool { false } + fn memory_limit(&self) -> usize { usize::max_value() } + fn is_precompiled_contract_enabled(&self, address: &Address) -> bool { + match address.low_u64() { + 0x1 | 0x2 | 0x3 | 0x4 => true, + _ => false, + } + } + fn precompileds(&self) -> &'static [(Address, Option<&'static [u8]>, &'static Precompiled)] { + &EMBEDDED_PRECOMPILEDS + } +}