From ae66fa34105c9fef7a77f25324cda5a113e55b68 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Tue, 22 Mar 2022 16:14:12 +0800 Subject: [PATCH 1/2] add increment export snapshot --- cmd/db-exporter/README.md | 32 +++++++++++ cmd/db-exporter/src/main.rs | 106 ++++++++++++++++++++++++++++-------- scripts/import_snapshot.sh | 69 +++++++++++++++++++++++ scripts/release.sh | 1 + 4 files changed, 185 insertions(+), 23 deletions(-) create mode 100644 scripts/import_snapshot.sh diff --git a/cmd/db-exporter/README.md b/cmd/db-exporter/README.md index b741b43e41..c7fbd37f1f 100644 --- a/cmd/db-exporter/README.md +++ b/cmd/db-exporter/README.md @@ -68,3 +68,35 @@ OPTIONS: ARGS: Verify type: Basic, Consensus, Full, None, eg [possible values: Basic, Consensus, Full, None] ``` +starcoin_db_export export_snapshot +```shell +./starcoin_db_exporter export-snapshot -i ~/.starcoin/main -n main -o ~/snapshot +USAGE: + starcoin_db_exporter export-snapshot [OPTIONS] --db-path --net --output + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + -i, --db-path starcoin node db path. like ~/.starcoin/main + -t, --increment enable increment export snapshot + -n, --net Chain Network, like main, proxima + -o, --output output dir, like ~/, manifest.csv will write in output dir +``` + +starcoin_db_export apply_snapshot +```shell +./starcoin_db_exporter apply-snapshot -i ~/snapshot -n main -o ~/.starcoin/main +USAGE: + starcoin_db_exporter apply-snapshot --input-path --net --to-path + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + -i, --input-path input_path, manifest.csv in this dir + -n, --net Chain Network + -o, --to-path starcoin node db path. like ~/.starcoin/main +``` diff --git a/cmd/db-exporter/src/main.rs b/cmd/db-exporter/src/main.rs index deac3ef4b1..6d1786c156 100644 --- a/cmd/db-exporter/src/main.rs +++ b/cmd/db-exporter/src/main.rs @@ -39,8 +39,10 @@ use starcoin_types::startup_info::{SnapshotRange, StartupInfo}; use starcoin_types::state_set::{AccountStateSet, ChainStateSet}; use starcoin_types::transaction::Transaction; use starcoin_vm_types::genesis_config::ConsensusStrategy; +use std::collections::HashMap; use std::fmt::{Debug, Formatter}; use std::fs::File; +use std::fs::OpenOptions; use std::io::{BufRead, BufReader, Write}; use std::path::PathBuf; use std::str::FromStr; @@ -342,9 +344,9 @@ pub struct ExportSnapshotOptions { #[structopt(long, short = "i", parse(from_os_str))] /// starcoin node db path. like ~/.starcoin/main pub db_path: PathBuf, - /// export snapshot block number - #[structopt(long, short = "b")] - pub number: Option, + #[structopt(long, short = "t")] + /// enable increment export snapshot + pub increment: Option, } #[derive(Debug, StructOpt)] @@ -478,7 +480,7 @@ fn main() -> anyhow::Result<()> { } if let Cmd::ExportSnapshot(option) = cmd { - let result = export_snapshot(option.db_path, option.output, option.net); + let result = export_snapshot(option.db_path, option.output, option.net, option.increment); return result; } @@ -486,6 +488,7 @@ fn main() -> anyhow::Result<()> { let result = apply_snapshot(option.to_path, option.input_path, option.net); return result; } + Ok(()) } @@ -958,33 +961,39 @@ fn export_column( accumulator: MerkleAccumulator, output: PathBuf, column: ColumnFamilyName, - nums: u64, + start_num: u64, + num: u64, bar: ProgressBar, ) -> Result<()> { - let mut file = File::create(output.join(column))?; + // start_num > 1 increment export + let mut file = if start_num > 1 { + OpenOptions::new().append(true).open(output.join(column))? + } else { + File::create(output.join(column))? + }; let mut index = 1; let mut start_index = 0; bar.set_style( ProgressStyle::default_bar() .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), ); - while start_index < nums { - let max_size = if start_index + BATCH_SIZE <= nums { + while start_index < num { + let max_size = if start_index + BATCH_SIZE <= num { BATCH_SIZE } else { - nums - start_index + num - start_index }; match column { BLOCK_ACCUMULATOR_NODE_PREFIX_NAME | TRANSACTION_ACCUMULATOR_NODE_PREFIX_NAME => { - let ids = accumulator.get_leaves(start_index + 1, false, max_size)?; + let ids = accumulator.get_leaves(start_index + start_num, false, max_size)?; for hash in ids { writeln!(file, "{}", hash)?; } } BLOCK_PREFIX_NAME => { // will cache ids - let ids = accumulator.get_leaves(start_index + 1, false, max_size)?; + let ids = accumulator.get_leaves(start_index + start_num, false, max_size)?; for hash in ids { let block = storage .get_block(hash)? @@ -994,7 +1003,7 @@ fn export_column( } BLOCK_INFO_PREFIX_NAME => { // will cache ids - let ids = accumulator.get_leaves(start_index + 1, false, max_size)?; + let ids = accumulator.get_leaves(start_index + start_num, false, max_size)?; for hash in ids { let block_info = storage .get_block_info(hash)? @@ -1009,6 +1018,7 @@ fn export_column( }; start_index += max_size; bar.set_message(format!("export {} {}", column, index).as_str()); + println!("export {} percent {}", column, index * 100 / num); bar.inc(1); index += 1; } @@ -1027,6 +1037,7 @@ pub fn export_snapshot( from_dir: PathBuf, output: PathBuf, network: BuiltinNetworkID, + increment: Option, ) -> anyhow::Result<()> { let start_time = SystemTime::now(); let net = ChainNetwork::new_builtin(network); @@ -1052,19 +1063,63 @@ pub fn export_snapshot( None, ) .expect("create block chain should success."); - let num = chain.status().head().number(); - let cur_num = if num <= SNAP_GAP { num } else { num - SNAP_GAP }; + let block_num = chain.status().head().number(); + let cur_num = if block_num <= SNAP_GAP { + block_num + } else { + block_num - SNAP_GAP + }; let cur_block = chain .get_block_by_number(cur_num)? .ok_or_else(|| format_err!("get block by number {} error", cur_num))?; let chain = BlockChain::new(net.time_service(), cur_block.id(), storage.clone(), None) .expect("create block chain should success."); let cur_num = chain.epoch().start_block_number(); - println!( - "chain height {} snapshot block height {}", - chain_info.head().number(), - cur_num - ); + + // increment export read num + let inc_export = increment.unwrap_or(false); + let mut old_snapshot_nums: HashMap = HashMap::new(); + if inc_export { + let reader = BufReader::new(File::open(output.join("manifest.csv"))?); + for record in reader.lines() { + let record = record?; + let str_list: Vec<&str> = record.split(' ').collect(); + if str_list.len() != 3 { + println!("manifest.csv {} error", record); + std::process::exit(1); + } + let column = str_list[0].to_string(); + let num = str_list[1].parse::()?; + old_snapshot_nums.insert(column, num); + } + if old_snapshot_nums.len() != 5 { + println!("increment export snapshot manifest.cvs error"); + std::process::exit(1); + } + let old_block_num = *old_snapshot_nums.get(BLOCK_PREFIX_NAME).ok_or_else(|| { + format_err!( + "increment export snapshot get {} number error", + BLOCK_PREFIX_NAME + ) + })?; + if old_block_num + BLOCK_GAP >= cur_num { + println!("increment snapshot gap too small"); + return Ok(()); + } + println!( + "chain height {} snapshot block cur height {} old height {}", + chain_info.head().number(), + cur_num, + old_block_num + ); + } else { + println!( + "chain height {} snapshot block height {}", + chain_info.head().number(), + cur_num + ); + } + let block = chain .get_block_by_number(cur_num)? .ok_or_else(|| format_err!("get block by number {} error", cur_num))?; @@ -1089,7 +1144,7 @@ pub fn export_snapshot( txn_accumulator_info.accumulator_root, )); let mbar = MultiProgress::new(); - for (column, nums, _hash) in mainfest_list.clone() { + for (column, num_record, _hash) in mainfest_list.clone() { let accumulator = match column { BLOCK_ACCUMULATOR_NODE_PREFIX_NAME | BLOCK_PREFIX_NAME | BLOCK_INFO_PREFIX_NAME => { MerkleAccumulator::new_with_info( @@ -1106,11 +1161,15 @@ pub fn export_snapshot( std::process::exit(1); } }; + let old_start_num = *old_snapshot_nums.get(column).unwrap_or(&0); + let num = num_record - old_start_num; + let start_num = old_start_num + 1; let storage2 = storage.clone(); let output2 = output.clone(); - let bar = mbar.add(ProgressBar::new(nums / BATCH_SIZE)); - let handle = - thread::spawn(move || export_column(storage2, accumulator, output2, column, nums, bar)); + let bar = mbar.add(ProgressBar::new(num / BATCH_SIZE)); + let handle = thread::spawn(move || { + export_column(storage2, accumulator, output2, column, start_num, num, bar) + }); handles.push(handle); } @@ -1152,6 +1211,7 @@ pub fn export_snapshot( for handle in handles { handle.join().unwrap().unwrap(); } + mainfest_list.push((STATE_NODE_PREFIX_NAME, nums, state_root)); // save manifest diff --git a/scripts/import_snapshot.sh b/scripts/import_snapshot.sh new file mode 100644 index 0000000000..32dcb558ca --- /dev/null +++ b/scripts/import_snapshot.sh @@ -0,0 +1,69 @@ +#!/bin/bash +function download() { + net=$1 + from_dir=$2 + compress_name=snapshot.tar.gz + url=https://s3.ap-northeast-1.amazonaws.com/main.starcoin.org/$net/$compress_name + for ((i = 0; i < 3; i++)); do + rm -f "$compress_name" + wget $url -P $from_dir + case_status=$? + if [ $case_status -eq 0 ]; then + echo -e "download $net $name succ" + break + fi + done + case_status=$? + if [ $case_status -ne 0 ]; then + return $case_status + fi + cd "$from_dir" + tar xzvf "$compress_name" -C $from_dir + case_status=$? + if [ $case_status -ne 0 ]; then + echo -e "tar $net $compress_name fail" + return $case_status + fi + cd - + return 0 +} + +function usage() { + echo -e "usage: import_snapshot.sh net from_dir to_dir" + echo -e "net is main, barnard, proxima, halley" + echo -e "from_dir like ~/snapshot" + echo -e "to_dir like ~/.starcoin/main，~/.starcoin/barnard" +} + +function import_snapshot() { + net=$1 + from_dir=$2 + to_dir=$3 + + download "$net" "$from_dir" + + ./starcoin_db_exporter apply-snapshot -i "$from_dir" -n "$net" -o "$to_dir" + case_status=$? + if [ $case_status -ne 0 ]; then + echo -e "apply-snapshot $net $from_dir fail" + exit $case_status + fi + echo -e "$net apply-snapshot succ" +} + +if [ $# != 3 ]; then + usage + exit 1 +fi +net=$1 +from_dir=$2 +to_dir=$3 +case $net in +"main" | "barnard" | "proxima" |"halley") + import_snapshot "$net" "$from_dir" "$to_dir" + ;; +*) + echo "$net not supported" + usage + ;; +esac \ No newline at end of file diff --git a/scripts/release.sh b/scripts/release.sh index 291336e3ad..d239a566ac 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -7,6 +7,7 @@ cp -v target/release/starcoin_generator starcoin-artifacts/ cp -v target/release/mpm starcoin-artifacts/ cp -v target/release/starcoin_db_exporter starcoin-artifacts/ cp -v scripts/import_net_block.sh starcoin_artifacts/ +cp -v scripts/import_snapshot.sh starcoin_artifacts/ if [ "$1" == "windows-latest" ]; then 7z a -r starcoin-$1.zip starcoin-artifacts else From bf0bfe5effb40a6dae1b9aa7628bcfaf20030967 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Tue, 22 Mar 2022 16:48:11 +0800 Subject: [PATCH 2/2] remove unused print --- cmd/db-exporter/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/db-exporter/src/main.rs b/cmd/db-exporter/src/main.rs index 6d1786c156..82ed36df99 100644 --- a/cmd/db-exporter/src/main.rs +++ b/cmd/db-exporter/src/main.rs @@ -1018,7 +1018,6 @@ fn export_column( }; start_index += max_size; bar.set_message(format!("export {} {}", column, index).as_str()); - println!("export {} percent {}", column, index * 100 / num); bar.inc(1); index += 1; }