Replace turbopack://[project]/... sourcemap uris with file://...

This makes working with devtools more straightforward, reduces our own overhead when tracing stack frames in the error overlay, etc. Generated code from Turbopack or Next.js still use `turbopack://[turbopack]` or `turbopack://[next]` respectively. Test Plan: CI. Update snapshots.
vercel · Oct 18, 2024 · b9c0f93 · b9c0f93
1 parent a1dbc07
commit b9c0f93
Show file tree

Hide file tree

Showing 39 changed files with 189 additions and 84 deletions.
diff --git a/crates/next-core/src/embed_js.rs b/crates/next-core/src/embed_js.rs
@@ -7,7 +7,7 @@ pub const VIRTUAL_PACKAGE_NAME: &str = "@vercel/turbopack-next";
 #[turbo_tasks::function]
 pub(crate) fn next_js_fs() -> Vc<Box<dyn FileSystem>> {
     // [TODO]: macro need to be refactored to be used via turbopack-binding
-    turbo_tasks_fs::embed_directory!("next", "$CARGO_MANIFEST_DIR/js/src")
+    turbo_tasks_fs::embed_directory!("turbopack", "next", "$CARGO_MANIFEST_DIR/js/src")
 }
 
 #[turbo_tasks::function]

diff --git a/turbopack/crates/turbo-tasks-fs/examples/hash_directory.rs b/turbopack/crates/turbo-tasks-fs/examples/hash_directory.rs
@@ -13,7 +13,7 @@ use sha2::{Digest, Sha256};
 use turbo_tasks::{util::FormatDuration, RcStr, ReadConsistency, TurboTasks, UpdateInfo, Vc};
 use turbo_tasks_fs::{
     register, DirectoryContent, DirectoryEntry, DiskFileSystem, FileContent, FileSystem,
-    FileSystemPath,
+    FileSystemPath, UriScheme,
 };
 use turbo_tasks_memory::MemoryBackend;
 
@@ -31,7 +31,12 @@ async fn main() -> Result<()> {
     let task = tt.spawn_root_task(|| {
         Box::pin(async {
             let root = current_dir().unwrap().to_str().unwrap().into();
-            let disk_fs = DiskFileSystem::new("project".into(), root, vec![]);
+            let disk_fs = DiskFileSystem::new(
+                UriScheme::Custom("turbopack".into()).cell(),
+                "project".into(),
+                root,
+                vec![],
+            );
             disk_fs.await?.start_watching(None).await?;
 
             // Smart Pointer cast

diff --git a/turbopack/crates/turbo-tasks-fs/examples/hash_glob.rs b/turbopack/crates/turbo-tasks-fs/examples/hash_glob.rs
@@ -13,7 +13,7 @@ use sha2::{Digest, Sha256};
 use turbo_tasks::{util::FormatDuration, RcStr, ReadConsistency, TurboTasks, UpdateInfo, Vc};
 use turbo_tasks_fs::{
     glob::Glob, register, DirectoryEntry, DiskFileSystem, FileContent, FileSystem, FileSystemPath,
-    ReadGlobResult,
+    ReadGlobResult, UriScheme,
 };
 use turbo_tasks_memory::MemoryBackend;
 
@@ -28,7 +28,12 @@ async fn main() -> Result<()> {
     let task = tt.spawn_root_task(|| {
         Box::pin(async {
             let root = current_dir().unwrap().to_str().unwrap().into();
-            let disk_fs = DiskFileSystem::new("project".into(), root, vec![]);
+            let disk_fs = DiskFileSystem::new(
+                UriScheme::Custom("turbopack".into()).cell(),
+                "project".into(),
+                root,
+                vec![],
+            );
             disk_fs.await?.start_watching(None).await?;
 
             // Smart Pointer cast

diff --git a/turbopack/crates/turbo-tasks-fs/src/embed/dir.rs b/turbopack/crates/turbo-tasks-fs/src/embed/dir.rs
@@ -4,24 +4,26 @@ pub use ::include_dir::{
 use anyhow::Result;
 use turbo_tasks::{RcStr, Vc};
 
-use crate::{embed::EmbeddedFileSystem, DiskFileSystem, FileSystem};
+use crate::{embed::EmbeddedFileSystem, DiskFileSystem, FileSystem, UriScheme};
 
 #[turbo_tasks::function]
 pub async fn directory_from_relative_path(
+    uri_scheme: Vc<UriScheme>,
     name: RcStr,
     path: RcStr,
 ) -> Result<Vc<Box<dyn FileSystem>>> {
-    let disk_fs = DiskFileSystem::new(name, path, vec![]);
+    let disk_fs = DiskFileSystem::new(uri_scheme, name, path, vec![]);
     disk_fs.await?.start_watching(None).await?;
 
     Ok(Vc::upcast(disk_fs))
 }
 
 pub fn directory_from_include_dir(
+    uri_scheme: RcStr,
     name: RcStr,
     dir: &'static include_dir::Dir<'static>,
 ) -> Vc<Box<dyn FileSystem>> {
-    Vc::upcast(EmbeddedFileSystem::new(name, dir))
+    Vc::upcast(EmbeddedFileSystem::new(uri_scheme, name, dir))
 }
 
 /// Returns an embedded [Vc<Box<dyn FileSystem>>] for the given path.
@@ -37,39 +39,43 @@ pub fn directory_from_include_dir(
 /// only the directory path will be embedded into the binary.
 #[macro_export]
 macro_rules! embed_directory {
-    ($name:tt, $path:tt) => {{        // make sure the path contains `$CARGO_MANIFEST_DIR`
+    ($uri_scheme:tt, $name:tt, $path:tt) => {{        // make sure the path contains `$CARGO_MANIFEST_DIR`
         assert!($path.contains("$CARGO_MANIFEST_DIR"));
         // make sure `CARGO_MANIFEST_DIR` is the only env variable in the path
         assert!(!$path.replace("$CARGO_MANIFEST_DIR", "").contains('$'));
 
-        turbo_tasks_fs::embed_directory_internal!($name, $path)
+        turbo_tasks_fs::embed_directory_internal!($uri_scheme, $name, $path)
     }};
 }
 
 #[cfg(feature = "dynamic_embed_contents")]
 #[macro_export]
 #[doc(hidden)]
 macro_rules! embed_directory_internal {
-    ($name:tt, $path:tt) => {{
+    ($uri_scheme:tt, $name:tt, $path:tt) => {{
         // make sure the types the `include_dir!` proc macro refers to are in scope
         use turbo_tasks_fs::embed::include_dir;
 
         let path = $path.replace("$CARGO_MANIFEST_DIR", env!("CARGO_MANIFEST_DIR"));
 
-        turbo_tasks_fs::embed::directory_from_relative_path($name.to_string(), path)
+        turbo_tasks_fs::embed::directory_from_relative_path(
+            $uri_scheme.into(),
+            $name.to_string(),
+            path,
+        )
     }};
 }
 
 #[cfg(not(feature = "dynamic_embed_contents"))]
 #[macro_export]
 #[doc(hidden)]
 macro_rules! embed_directory_internal {
-    ($name:tt, $path:tt) => {{
+    ($uri_scheme:tt, $name:tt, $path:tt) => {{
         // make sure the types the `include_dir!` proc macro refers to are in scope
         use turbo_tasks_fs::embed::include_dir;
 
         static dir: include_dir::Dir<'static> = turbo_tasks_fs::embed::include_dir!($path);
 
-        turbo_tasks_fs::embed::directory_from_include_dir($name.into(), &dir)
+        turbo_tasks_fs::embed::directory_from_include_dir($uri_scheme.into(), $name.into(), &dir)
     }};
 }
diff --git a/turbopack/crates/turbo-tasks-fs/src/embed/file.rs b/turbopack/crates/turbo-tasks-fs/src/embed/file.rs
@@ -4,10 +4,11 @@ use anyhow::{Context, Result};
 use dunce::canonicalize;
 use turbo_tasks::{RcStr, Vc};
 
-use crate::{DiskFileSystem, File, FileContent, FileSystem};
+use crate::{DiskFileSystem, File, FileContent, FileSystem, UriScheme};
 
 #[turbo_tasks::function]
 pub async fn content_from_relative_path(
+    uri_scheme: Vc<UriScheme>,
     package_path: RcStr,
     path: RcStr,
 ) -> Result<Vc<FileContent>> {
@@ -19,6 +20,7 @@ pub async fn content_from_relative_path(
     let path = resolved_path.file_name().unwrap().to_str().unwrap();
 
     let disk_fs = DiskFileSystem::new(
+        uri_scheme,
         root_path.to_string_lossy().into(),
         root_path.to_string_lossy().into(),
         vec![],

diff --git a/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs b/turbopack/crates/turbo-tasks-fs/src/embed/fs.rs
@@ -13,11 +13,21 @@ pub struct EmbeddedFileSystem {
     name: RcStr,
     #[turbo_tasks(trace_ignore)]
     dir: &'static Dir<'static>,
+    uri_scheme: RcStr,
 }
 
 impl EmbeddedFileSystem {
-    pub(super) fn new(name: RcStr, dir: &'static Dir<'static>) -> Vc<EmbeddedFileSystem> {
-        EmbeddedFileSystem { name, dir }.cell()
+    pub(super) fn new(
+        uri_scheme: RcStr,
+        name: RcStr,
+        dir: &'static Dir<'static>,
+    ) -> Vc<EmbeddedFileSystem> {
+        EmbeddedFileSystem {
+            name,
+            dir,
+            uri_scheme,
+        }
+        .cell()
     }
 }
 
@@ -100,6 +110,15 @@ impl FileSystem for EmbeddedFileSystem {
 
         Ok(FileMeta::default().cell())
     }
+
+    #[turbo_tasks::function(fs)]
+    async fn file_uri(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<RcStr>> {
+        let path_str = &*fs_path.await?.path;
+        Ok(Vc::cell(RcStr::from(format!(
+            "{}://[{}]/{}",
+            self.uri_scheme, self.name, path_str
+        ))))
+    }
 }
 
 #[turbo_tasks::value_impl]

diff --git a/turbopack/crates/turbo-tasks-fs/src/lib.rs b/turbopack/crates/turbo-tasks-fs/src/lib.rs
@@ -193,11 +193,15 @@ pub trait FileSystem: ValueToString {
         target: Vc<LinkContent>,
     ) -> Vc<Completion>;
     fn metadata(self: Vc<Self>, fs_path: Vc<FileSystemPath>) -> Vc<FileMeta>;
+    fn file_uri(self: Vc<Self>, _fs_path: Vc<FileSystemPath>) -> Vc<RcStr> {
+        unimplemented!();
+    }
 }
 
 #[turbo_tasks::value(cell = "new", eq = "manual")]
 pub struct DiskFileSystem {
     pub name: RcStr,
+    pub uri_scheme: Vc<UriScheme>,
     pub root: RcStr,
     #[turbo_tasks(debug_ignore, trace_ignore)]
     #[serde(skip)]
@@ -415,6 +419,12 @@ pub fn path_to_key(path: impl AsRef<Path>) -> String {
     path.as_ref().to_string_lossy().to_string()
 }
 
+#[turbo_tasks::value(shared)]
+pub enum UriScheme {
+    File,
+    Custom(RcStr),
+}
+
 #[turbo_tasks::value_impl]
 impl DiskFileSystem {
     /// Create a new instance of `DiskFileSystem`.
@@ -426,12 +436,18 @@ impl DiskFileSystem {
     ///   be a full path, since it is possible that root & project dir is different and requires to
     ///   ignore specific subpaths from each.
     #[turbo_tasks::function]
-    pub async fn new(name: RcStr, root: RcStr, ignored_subpaths: Vec<RcStr>) -> Result<Vc<Self>> {
+    pub async fn new(
+        uri_scheme: Vc<UriScheme>,
+        name: RcStr,
+        root: RcStr,
+        ignored_subpaths: Vec<RcStr>,
+    ) -> Result<Vc<Self>> {
         mark_stateful();
 
         let instance = DiskFileSystem {
             name,
             root,
+            uri_scheme,
             mutex_map: Default::default(),
             invalidation_lock: Default::default(),
             invalidator_map: Arc::new(InvalidatorMap::new()),
@@ -886,6 +902,22 @@ impl FileSystem for DiskFileSystem {
 
         Ok(FileMeta::cell(meta.into()))
     }
+
+    #[turbo_tasks::function(fs)]
+    async fn file_uri(&self, fs_path: Vc<FileSystemPath>) -> Result<Vc<RcStr>> {
+        Ok(Vc::cell(RcStr::from(match &*self.uri_scheme.await? {
+            UriScheme::File => {
+                format!(
+                    "file://{}",
+                    sys_to_unix(&self.to_sys_path(fs_path).await?.to_string_lossy())
+                )
+            }
+            UriScheme::Custom(uri_scheme) => {
+                let path_str = &*fs_path.await?.path;
+                format!("{}://[{}]/{}", uri_scheme, self.name, path_str)
+            }
+        })))
+    }
 }
 
 #[turbo_tasks::value_impl]
@@ -1301,6 +1333,12 @@ impl FileSystemPath {
             Cow::Owned(path) => path.cell(),
         })
     }
+
+    #[turbo_tasks::function]
+    pub async fn uri(self: Vc<Self>) -> Result<Vc<RcStr>> {
+        let this = self.await?;
+        Ok(this.fs.file_uri(self))
+    }
 }
 
 impl Display for FileSystemPath {

diff --git a/turbopack/crates/turbopack-cli/src/embed_js.rs b/turbopack/crates/turbopack-cli/src/embed_js.rs
@@ -3,7 +3,7 @@ use turbo_tasks_fs::{embed_directory, FileContent, FileSystem, FileSystemPath};
 
 #[turbo_tasks::function]
 fn embed_fs() -> Vc<Box<dyn FileSystem>> {
-    embed_directory!("turbopack-cli", "$CARGO_MANIFEST_DIR/js/src")
+    embed_directory!("turbopack", "turbopack-cli", "$CARGO_MANIFEST_DIR/js/src")
 }
 
 #[turbo_tasks::function]

diff --git a/turbopack/crates/turbopack-core/src/source_map/mod.rs b/turbopack/crates/turbopack-core/src/source_map/mod.rs
@@ -359,17 +359,16 @@ impl SourceMap {
         ) -> Result<(Arc<str>, Arc<str>)> {
             Ok(
                 if let Some(path) = *origin.parent().try_join((&*source_request).into()).await? {
-                    let path_str = path.to_string().await?;
-                    let source = format!("{SOURCE_MAP_PREFIX}{}", path_str);
+                    let source = &*path.uri().await?;
                     let source_content = if let Some(source_content) = source_content {
                         source_content
                     } else if let FileContent::Content(file) = &*path.read().await? {
                         let text = file.content().to_str()?;
                         text.to_string().into()
                     } else {
-                        format!("unable to read source {path_str}").into()
+                        format!("unable to read source {source}").into()
                     };
-                    (source.into(), source_content)
+                    (source.to_string().into(), source_content)
                 } else {
                     let origin_str = origin.to_string().await?;
                     static INVALID_REGEX: Lazy<Regex> =

diff --git a/turbopack/crates/turbopack-css/src/parse.rs b/turbopack/crates/turbopack-css/src/parse.rs
@@ -9,7 +9,10 @@ pub struct InlineSourcesContentConfig {}
 impl SourceMapGenConfig for InlineSourcesContentConfig {
     fn file_name_to_source(&self, f: &FileName) -> String {
         match f {
-            FileName::Custom(s) => format!("{SOURCE_MAP_PREFIX}{s}"),
+            FileName::Custom(s) => {
+                dbg!(s);
+                format!("{SOURCE_MAP_PREFIX}{s}")
+            }
             _ => f.to_string(),
         }
     }