[Turbopack] add rocksdb (#71688)

### What?

switches the database for persistent caching from lmdb to rocksdb.

### Why?

* No preallocated sparse file which cause problems
* Compression support
* Multiple files instead of a big single file
* Better performance

.github/workflows/build_and_test.yml

@@ -190,6 +190,7 @@ jobs:
           - '--scenario=heavy-npm-deps-build --page=homepage'
     uses: ./.github/workflows/build_reusable.yml
     with:
+      skipNativeBuild: 'yes'
       afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }}
       stepName: 'devlow-bench-${{ matrix.mode }}-${{ matrix.selector }}'
     secrets: inherit
@@ -295,6 +296,7 @@ jobs:
 
     uses: ./.github/workflows/build_reusable.yml
     with:
+      skipNativeBuild: 'yes'
       afterBuild: rustup target add wasm32-unknown-unknown && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && node ./scripts/normalize-version-bump.js && turbo run build-wasm -- --target nodejs && git checkout . && mv crates/wasm/pkg crates/wasm/pkg-nodejs && node ./scripts/setup-wasm.mjs && NEXT_TEST_MODE=start TEST_WASM=true node run-tests.js test/production/pages-dir/production/test/index.test.ts test/e2e/streaming-ssr/index.test.ts
       stepName: 'test-next-swc-wasm'
     secrets: inherit

turbopack/crates/turbo-tasks-backend/Cargo.toml

@@ -13,9 +13,11 @@ bench = false
 workspace = true
 
 [features]
-default = []
+default = ["rocksdb"]
 verify_serialization = []
 trace_aggregation_update = []
+lmdb = ["dep:lmdb-rkv"]
+rocksdb = ["dep:rocksdb"]
 
 [dependencies]
 anyhow = { workspace = true }
@@ -27,12 +29,13 @@ dashmap = { workspace = true, features = ["raw-api"]}
 either = { workspace = true }
 hashbrown = { workspace = true, features = ["raw"] }
 indexmap = { workspace = true }
-lmdb-rkv = "0.14.0"
+lmdb-rkv = { version = "0.14.0", optional = true }
 once_cell = { workspace = true }
 parking_lot = { workspace = true }
 pot = "3.0.0"
 rand = { workspace = true }
 rayon = { workspace = true }
+rocksdb = { version = "0.22.0", optional = true}
 rustc-hash = { workspace = true }
 serde = { workspace = true }
 serde_path_to_error = { workspace = true }

turbopack/crates/turbo-tasks-backend/src/database/mod.rs

@@ -2,14 +2,21 @@ mod by_key_space;
 pub mod db_versioning;
 pub mod fresh_db_optimization;
 pub mod key_value_database;
+#[cfg(feature = "lmdb")]
 pub mod lmdb;
 pub mod noop_kv;
 pub mod read_transaction_cache;
+#[cfg(feature = "rocksdb")]
+pub mod rocksdb;
 mod startup_cache;
 
 pub use db_versioning::handle_db_versioning;
 pub use fresh_db_optimization::{is_fresh, FreshDbOptimization};
+#[cfg(feature = "lmdb")]
+pub use lmdb::LmbdKeyValueDatabase;
 #[allow(unused_imports)]
 pub use noop_kv::NoopKvDb;
 pub use read_transaction_cache::ReadTransactionCache;
+#[cfg(feature = "rocksdb")]
+pub use rocksdb::RocksDbKeyValueDatabase;
 pub use startup_cache::StartupCacheLayer;