Merge branch 'master' into wasmtime

chain/chain/src/store.rs

@@ -1870,6 +1870,10 @@ impl<'a> ChainStoreUpdate<'a> {
 
     pub fn update_tail(&mut self, height: BlockHeight) {
         self.tail = Some(height);
+        if self.chunk_tail.is_none() {
+            // For consistency, Chunk Tail should be set if Tail is set
+            self.chunk_tail = Some(self.get_genesis_height());
+        }
     }
 
     pub fn update_chunk_tail(&mut self, height: BlockHeight) {
@@ -2226,29 +2230,29 @@ impl<'a> ChainStoreUpdate<'a> {
         self.store_updates.push(store_update);
     }
 
+    fn write_col_misc<T: BorshSerialize>(
+        store_update: &mut StoreUpdate,
+        key: &[u8],
+        value: &mut Option<T>,
+    ) -> Result<(), Error> {
+        if let Some(t) = value.take() {
+            store_update.set_ser(ColBlockMisc, key, &t)?;
+        }
+        Ok(())
+    }
+
     fn finalize(&mut self) -> Result<StoreUpdate, Error> {
         let mut store_update = self.store().store_update();
-        if let Some(t) = self.head.take() {
-            store_update.set_ser(ColBlockMisc, HEAD_KEY, &t).map_err::<Error, _>(|e| e.into())?;
-        }
-        if let Some(t) = self.tail.take() {
-            store_update.set_ser(ColBlockMisc, TAIL_KEY, &t)?
-        }
-        if let Some(t) = self.header_head.take() {
-            store_update
-                .set_ser(ColBlockMisc, HEADER_HEAD_KEY, &t)
-                .map_err::<Error, _>(|e| e.into())?;
-        }
-        if let Some(t) = self.sync_head.take() {
-            store_update
-                .set_ser(ColBlockMisc, SYNC_HEAD_KEY, &t)
-                .map_err::<Error, _>(|e| e.into())?;
-        }
-        if let Some(t) = self.largest_target_height {
-            store_update
-                .set_ser(ColBlockMisc, LARGEST_TARGET_HEIGHT_KEY, &t)
-                .map_err::<Error, _>(|e| e.into())?;
-        }
+        Self::write_col_misc(&mut store_update, HEAD_KEY, &mut self.head)?;
+        Self::write_col_misc(&mut store_update, TAIL_KEY, &mut self.tail)?;
+        Self::write_col_misc(&mut store_update, CHUNK_TAIL_KEY, &mut self.chunk_tail)?;
+        Self::write_col_misc(&mut store_update, SYNC_HEAD_KEY, &mut self.sync_head)?;
+        Self::write_col_misc(&mut store_update, HEADER_HEAD_KEY, &mut self.header_head)?;
+        Self::write_col_misc(
+            &mut store_update,
+            LARGEST_TARGET_HEIGHT_KEY,
+            &mut self.largest_target_height,
+        )?;
         for (hash, block) in self.chain_store_cache_update.blocks.iter() {
             let mut map =
                 match self.chain_store.get_all_block_hashes_by_height(block.header().height()) {
@@ -2627,7 +2631,6 @@ mod tests {
     use cached::Cached;
     use strum::IntoEnumIterator;
 
-    use near_chain_configs::GenesisConfig;
     use near_crypto::KeyType;
     use near_primitives::block::{Block, Tip};
     use near_primitives::errors::InvalidTxError;
@@ -2639,11 +2642,12 @@ mod tests {
 
     use crate::chain::check_refcount_map;
     use crate::store::{ChainStoreAccess, GCMode};
-    use crate::store_validator::StoreValidator;
     use crate::test_utils::KeyValueRuntime;
     use crate::{Chain, ChainGenesis, DoomslugThresholdMode};
 
     use near_store::DBCol;
+    #[cfg(feature = "expensive_tests")]
+    use {crate::store_validator::StoreValidator, near_chain_configs::GenesisConfig};
 
     fn get_chain() -> Chain {
         get_chain_with_epoch_length(10)
@@ -3012,6 +3016,7 @@ mod tests {
         test_clear_old_data_too_many_heights_common(87);
     }
 
+    #[cfg(feature = "expensive_tests")]
     fn test_clear_old_data_too_many_heights_common(gc_blocks_limit: NumBlocks) {
         let mut chain = get_chain_with_epoch_length(1);
         let genesis = chain.get_block_by_height(0).unwrap().clone();

chain/chain/src/store_validator/validate.rs

@@ -111,16 +111,24 @@ macro_rules! unwrap_or_err_db {
 // All validations start here
 
 pub(crate) fn head_tail_validity(sv: &mut StoreValidator) -> Result<(), StoreValidatorError> {
-    let tail = unwrap_or_err!(
+    let mut tail = sv.config.genesis_height;
+    let mut chunk_tail = sv.config.genesis_height;
+    let tail_db = unwrap_or_err!(
         sv.store.get_ser::<BlockHeight>(ColBlockMisc, TAIL_KEY),
         "Can't get Tail from storage"
-    )
-    .unwrap_or(sv.config.genesis_height);
-    let chunk_tail = unwrap_or_err!(
+    );
+    let chunk_tail_db = unwrap_or_err!(
         sv.store.get_ser::<BlockHeight>(ColBlockMisc, CHUNK_TAIL_KEY),
         "Can't get Chunk Tail from storage"
-    )
-    .unwrap_or(sv.config.genesis_height);
+    );
+    if tail_db.is_none() && chunk_tail_db.is_some() || tail_db.is_some() && chunk_tail_db.is_none()
+    {
+        err!("Tail is {:?} and Chunk Tail is {:?}", tail_db, chunk_tail_db);
+    }
+    if tail_db.is_some() && chunk_tail_db.is_some() {
+        tail = tail_db.unwrap();
+        chunk_tail = chunk_tail_db.unwrap();
+    }
     let head = unwrap_or_err_db!(
         sv.store.get_ser::<Tip>(ColBlockMisc, HEAD_KEY),
         "Can't get Head from storage"
@@ -256,7 +264,7 @@ pub(crate) fn chunk_tail_validity(
     check_cached!(sv.inner.is_misc_set, "misc");
     let chunk_tail = sv.inner.chunk_tail;
     let height = shard_chunk.header.inner.height_created;
-    if height < chunk_tail {
+    if height != sv.config.genesis_height && height < chunk_tail {
         err!(
             "Invalid ShardChunk stored, chunk_tail = {:?}, ShardChunk = {:?}",
             chunk_tail,

chain/chain/src/test_utils.rs

@@ -629,6 +629,7 @@ impl RuntimeAdapter for KeyValueRuntime {
                         logs: vec![],
                         receipt_ids: new_receipt_hashes,
                         gas_burnt: 0,
+                        tokens_burnt: 0,
                     },
                 });
             }

chain/chain/src/types.rs

@@ -580,6 +580,7 @@ mod tests {
                 logs: vec!["outcome1".to_string()],
                 receipt_ids: vec![hash(&[1])],
                 gas_burnt: 100,
+                tokens_burnt: 10000,
             },
         };
         let outcome2 = ExecutionOutcomeWithId {
@@ -589,6 +590,7 @@ mod tests {
                 logs: vec!["outcome2".to_string()],
                 receipt_ids: vec![],
                 gas_burnt: 0,
+                tokens_burnt: 0,
             },
         };
         let outcomes = vec![outcome1, outcome2];

chain/chunks/src/lib.rs

@@ -1412,22 +1412,26 @@ impl ShardsManager {
 mod test {
     use crate::test_utils::SealsManagerTestFixture;
     use crate::{
-        ChunkRequestInfo, Seal, SealsManager, ShardsManager, ACCEPTING_SEAL_PERIOD_MS,
-        CHUNK_REQUEST_RETRY_MS, NUM_PARTS_REQUESTED_IN_SEAL, PAST_SEAL_HEIGHT_HORIZON,
+        ChunkRequestInfo, Seal, SealsManager, ShardsManager, CHUNK_REQUEST_RETRY_MS,
+        NUM_PARTS_REQUESTED_IN_SEAL, PAST_SEAL_HEIGHT_HORIZON,
     };
     use near_chain::test_utils::KeyValueRuntime;
-    use near_chain::{ChainStore, RuntimeAdapter};
-    use near_crypto::KeyType;
-    use near_logger_utils::init_test_logger;
     use near_network::test_utils::MockNetworkAdapter;
-    use near_primitives::hash::{hash, CryptoHash};
-    use near_primitives::merkle::merklize;
-    use near_primitives::sharding::{ChunkHash, ReedSolomonWrapper};
-    use near_primitives::validator_signer::InMemoryValidatorSigner;
+    use near_primitives::hash::hash;
+    use near_primitives::sharding::ChunkHash;
     use near_store::test_utils::create_test_store;
     use std::sync::Arc;
     use std::time::{Duration, Instant};
 
+    #[cfg(feature = "expensive_tests")]
+    use {
+        crate::ACCEPTING_SEAL_PERIOD_MS, near_chain::ChainStore, near_chain::RuntimeAdapter,
+        near_crypto::KeyType, near_logger_utils::init_test_logger,
+        near_primitives::hash::CryptoHash, near_primitives::merkle::merklize,
+        near_primitives::sharding::ReedSolomonWrapper,
+        near_primitives::validator_signer::InMemoryValidatorSigner,
+    };
+
     /// should not request partial encoded chunk from self
     #[test]
     fn test_request_partial_encoded_chunk_from_self() {

chain/client/tests/catching_up.rs

@@ -437,7 +437,7 @@ mod tests {
                 key_pairs.clone(),
                 validator_groups,
                 true,
-                1800,
+                3500,
                 false,
                 false,
                 5,
@@ -600,7 +600,7 @@ mod tests {
             );
             *connectors.write().unwrap() = conn;
 
-            near_network::test_utils::wait_or_panic(240000);
+            near_network::test_utils::wait_or_panic(480000);
         })
         .unwrap();
     }

chain/client/tests/process_blocks.rs

@@ -788,6 +788,7 @@ fn test_gc_with_epoch_length_common(epoch_length: NumBlocks) {
                 .is_ok());
         }
     }
+    assert_eq!(env.clients[0].chain.store().chunk_tail().unwrap(), epoch_length - 1);
     assert!(check_refcount_map(&mut env.clients[0].chain).is_ok());
 }
 

core/primitives/src/transaction.rs

@@ -12,7 +12,7 @@ use crate::errors::TxExecutionError;
 use crate::hash::{hash, CryptoHash};
 use crate::logging;
 use crate::merkle::MerklePath;
-use crate::serialize::{base64_format, u128_dec_format_compatible};
+use crate::serialize::{base64_format, u128_dec_format, u128_dec_format_compatible};
 use crate::types::{AccountId, Balance, Gas, Nonce};
 
 pub type LogEntry = String;
@@ -233,6 +233,8 @@ impl Default for ExecutionStatus {
 struct PartialExecutionOutcome {
     pub receipt_ids: Vec<CryptoHash>,
     pub gas_burnt: Gas,
+    #[serde(with = "u128_dec_format")]
+    pub tokens_burnt: Balance,
     pub status: PartialExecutionStatus,
 }
 
@@ -265,8 +267,13 @@ pub struct ExecutionOutcome {
     pub receipt_ids: Vec<CryptoHash>,
     /// The amount of the gas burnt by the given transaction or receipt.
     pub gas_burnt: Gas,
+    /// The amount of tokens burnt corresponding to the burnt gas amount.
+    /// This value doesn't always equal to the `gas_burnt` multiplied by the gas price, because
+    /// the prepaid gas price might be lower than the actual gas price and it creates a deficit.
+    pub tokens_burnt: Balance,
     /// Execution status. Contains the result in case of successful execution.
-    /// Should be the latest field since contains unparsable by light client ExecutionStatus::Failure
+    /// NOTE: Should be the latest field since it contains unparsable by light client
+    /// ExecutionStatus::Failure
     pub status: ExecutionStatus,
 }
 
@@ -276,6 +283,7 @@ impl ExecutionOutcome {
             &PartialExecutionOutcome {
                 receipt_ids: self.receipt_ids.clone(),
                 gas_burnt: self.gas_burnt,
+                tokens_burnt: self.tokens_burnt,
                 status: self.status.clone().into(),
             }
             .try_to_vec()
@@ -294,6 +302,7 @@ impl fmt::Debug for ExecutionOutcome {
             .field("logs", &format_args!("{}", logging::pretty_vec(&self.logs)))
             .field("receipt_ids", &format_args!("{}", logging::pretty_vec(&self.receipt_ids)))
             .field("burnt_gas", &self.gas_burnt)
+            .field("tokens_burnt", &self.tokens_burnt)
             .field("status", &self.status)
             .finish()
     }
@@ -434,6 +443,7 @@ mod tests {
             logs: vec!["123".to_string(), "321".to_string()],
             receipt_ids: vec![],
             gas_burnt: 123,
+            tokens_burnt: 1234000,
         };
         let hashes = outcome.to_hashes();
         assert_eq!(hashes.len(), 3);

core/primitives/src/version.rs

@@ -17,6 +17,6 @@ pub const DB_VERSION: DbVersion = 2;
 pub type ProtocolVersion = u32;
 
 /// Current latest version of the protocol.
-pub const PROTOCOL_VERSION: ProtocolVersion = 26;
+pub const PROTOCOL_VERSION: ProtocolVersion = 27;
 
 pub const FIRST_BACKWARD_COMPATIBLE_PROTOCOL_VERSION: ProtocolVersion = PROTOCOL_VERSION;

core/primitives/src/views.rs

@@ -800,6 +800,11 @@ pub struct ExecutionOutcomeView {
     pub receipt_ids: Vec<CryptoHash>,
     /// The amount of the gas burnt by the given transaction or receipt.
     pub gas_burnt: Gas,
+    /// The amount of tokens burnt corresponding to the burnt gas amount.
+    /// This value doesn't always equal to the `gas_burnt` multiplied by the gas price, because
+    /// the prepaid gas price might be lower than the actual gas price and it creates a deficit.
+    #[serde(with = "u128_dec_format")]
+    pub tokens_burnt: Balance,
     /// Execution status. Contains the result in case of successful execution.
     pub status: ExecutionStatusView,
 }
@@ -810,6 +815,7 @@ impl From<ExecutionOutcome> for ExecutionOutcomeView {
             logs: outcome.logs,
             receipt_ids: outcome.receipt_ids,
             gas_burnt: outcome.gas_burnt,
+            tokens_burnt: outcome.tokens_burnt,
             status: outcome.status.into(),
         }
     }

neard/res/genesis_config.json

@@ -1,5 +1,5 @@
 {
-  "protocol_version": 26,
+  "protocol_version": 27,
   "genesis_time": "1970-01-01T00:00:00.000000000Z",
   "chain_id": "sample",
   "genesis_height": 0,

neard/src/lib.rs

@@ -2,7 +2,7 @@ use std::fs;
 use std::path::Path;
 use std::sync::Arc;
 
-use actix::{Actor, Addr};
+use actix::{Actor, Addr, Arbiter};
 use log::{error, info};
 use tracing::trace;
 
@@ -135,14 +135,15 @@ pub fn start_with_config(
 
     config.network_config.verify();
 
-    let network_actor = PeerManagerActor::new(
-        store,
-        config.network_config,
-        client_actor.clone().recipient(),
-        view_client.clone().recipient(),
-    )
-    .unwrap()
-    .start();
+    let arbiter = Arbiter::new();
+
+    let client_actor1 = client_actor.clone().recipient();
+    let view_client1 = view_client.clone().recipient();
+    let network_config = config.network_config;
+
+    let network_actor = PeerManagerActor::start_in_arbiter(&arbiter, move |_ctx| {
+        PeerManagerActor::new(store, network_config, client_actor1, view_client1).unwrap()
+    });
 
     network_adapter.set_recipient(network_actor.recipient());
 

neard/tests/rpc_nodes.rs

@@ -465,7 +465,7 @@ fn outcome_view_to_hashes(outcome: &ExecutionOutcomeView) -> Vec<CryptoHash> {
         ExecutionStatusView::SuccessReceiptId(id) => PartialExecutionStatus::SuccessReceiptId(*id),
     };
     let mut result = vec![hash(
-        &(outcome.receipt_ids.clone(), outcome.gas_burnt, status)
+        &(outcome.receipt_ids.clone(), outcome.gas_burnt, outcome.tokens_burnt, status)
             .try_to_vec()
             .expect("Failed to serialize"),
     )];

pytest/tests/sanity/backward_compatible.py

@@ -22,9 +22,8 @@ def main():
         shutil.rmtree(node_root)
     subprocess.check_output('mkdir -p /tmp/near', shell=True)
 
-    # near_root, (stable_branch,
-    #             current_branch) = branches.prepare_ab_test("beta")
-    (near_root, (stable_branch, current_branch)) = ('../target/debug', ('beta', 'upgradability'))
+    near_root, (stable_branch,
+                current_branch) = branches.prepare_ab_test("beta")
 
     # Setup local network.
     subprocess.call([

pytest/tests/sanity/rpc_light_client_execution_outcome_proof.py

@@ -41,6 +41,7 @@ class Failure:
         'fields': [
             ['receipt_ids', [[32]]],
             ['gas_burnt', 'u64'],
+            ['tokens_burnt', 'u128'],
             ['status', PartialExecutionStatus],
         ]
     },
@@ -76,6 +77,7 @@ def serialize_execution_outcome_with_id(outcome, id):
     partial_outcome = PartialExecutionOutcome()
     partial_outcome.receipt_ids = [base58.b58decode(x) for x in outcome['receipt_ids']]
     partial_outcome.gas_burnt = outcome['gas_burnt']
+    partial_outcome.tokens_burnt = int(outcome['tokens_burnt'])
     execution_status = PartialExecutionStatus()
     if 'SuccessValue' in outcome['status']:
         execution_status.enum = 'successValue'
@@ -131,7 +133,7 @@ def check_transaction_outcome_proof(should_succeed, nonce):
     status = nodes[1].get_status()
     latest_block_hash = status['sync_info']['latest_block_hash']
     function_caller_key = nodes[0].signer_key
-    gas = 10000000000000000 if should_succeed else 1000
+    gas = 300000000000000 if should_succeed else 1000
 
     function_call_1_tx = transaction.sign_function_call_tx(
         function_caller_key, contract_key.account_id, 'setKeyValue',