diff --git a/Cargo.lock b/Cargo.lock index c1dbe6a7a5e0..caab97979f35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -772,7 +772,6 @@ dependencies = [ "hir-def", "hir-expand", "hir-ty", - "indexmap", "intern", "itertools 0.14.0", "ra-ap-rustc_type_ir", @@ -824,7 +823,6 @@ dependencies = [ "syntax-bridge", "test-fixture", "test-utils", - "text-size 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "thin-vec", "tracing", "triomphe", @@ -864,7 +862,6 @@ version = "0.0.0" dependencies = [ "arrayvec", "base-db", - "bitflags 2.9.4", "cov-mark", "either", "ena", @@ -890,7 +887,6 @@ dependencies = [ "rustc_apfloat", "salsa", "salsa-macros", - "scoped-tls", "smallvec", "span", "stdx", @@ -1085,7 +1081,6 @@ dependencies = [ "expect-test", "fst", "hir", - "indexmap", "itertools 0.14.0", "line-index 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "macros", @@ -1825,6 +1820,7 @@ dependencies = [ "indexmap", "intern", "paths", + "proc-macro-srv", "rustc-hash 2.1.1", "serde", "serde_derive", @@ -1849,9 +1845,7 @@ dependencies = [ "proc-macro-test", "ra-ap-rustc_lexer", "span", - "syntax-bridge", "temp-dir", - "tt", ] [[package]] @@ -2291,14 +2285,12 @@ dependencies = [ "ide-db", "ide-ssr", "indexmap", - "intern", "itertools 0.14.0", "load-cargo", "lsp-server 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", "lsp-types", "memchr", "mimalloc", - "nohash-hasher", "num_cpus", "oorandom", "parking_lot", @@ -2491,12 +2483,6 @@ dependencies = [ "protobuf", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" diff --git a/crates/hir-def/Cargo.toml b/crates/hir-def/Cargo.toml index abb4819a7672..d0ab03766403 100644 --- a/crates/hir-def/Cargo.toml +++ b/crates/hir-def/Cargo.toml @@ -27,7 +27,6 @@ tracing.workspace = true smallvec.workspace = true triomphe.workspace = true rustc_apfloat = "0.2.3" -text-size.workspace = true salsa.workspace = true salsa-macros.workspace = true query-group.workspace = true diff --git a/crates/hir-ty/Cargo.toml b/crates/hir-ty/Cargo.toml index 378a0f0382c3..fc2c0d5ac1ad 100644 --- a/crates/hir-ty/Cargo.toml +++ b/crates/hir-ty/Cargo.toml @@ -16,14 +16,12 @@ doctest = false cov-mark = "2.0.0" itertools.workspace = true arrayvec.workspace = true -bitflags.workspace = true smallvec.workspace = true ena = "0.14.3" either.workspace = true oorandom = "11.1.5" tracing.workspace = true rustc-hash.workspace = true -scoped-tls = "1.0.1" la-arena.workspace = true triomphe.workspace = true typed-arena = "2.0.2" diff --git a/crates/hir/Cargo.toml b/crates/hir/Cargo.toml index dfa39384320d..d2d43e0d86c9 100644 --- a/crates/hir/Cargo.toml +++ b/crates/hir/Cargo.toml @@ -20,7 +20,6 @@ itertools.workspace = true smallvec.workspace = true tracing.workspace = true triomphe.workspace = true -indexmap.workspace = true ra-ap-rustc_type_ir.workspace = true diff --git a/crates/ide-db/Cargo.toml b/crates/ide-db/Cargo.toml index b7148160182c..14b038ae403c 100644 --- a/crates/ide-db/Cargo.toml +++ b/crates/ide-db/Cargo.toml @@ -22,7 +22,6 @@ rustc-hash.workspace = true either.workspace = true itertools.workspace = true arrayvec.workspace = true -indexmap.workspace = true memchr = "2.7.5" salsa.workspace = true salsa-macros.workspace = true diff --git a/crates/proc-macro-api/Cargo.toml b/crates/proc-macro-api/Cargo.toml index dac8e0943576..63745b9f7493 100644 --- a/crates/proc-macro-api/Cargo.toml +++ b/crates/proc-macro-api/Cargo.toml @@ -24,10 +24,14 @@ indexmap.workspace = true paths = { workspace = true, features = ["serde1"] } tt.workspace = true stdx.workspace = true +proc-macro-srv = {workspace = true, optional = true} # span = {workspace = true, default-features = false} does not work span = { path = "../span", version = "0.0.0", default-features = false} intern.workspace = true +[features] +sysroot-abi = ["proc-macro-srv", "proc-macro-srv/sysroot-abi"] + [lints] workspace = true diff --git a/crates/proc-macro-api/src/legacy_protocol.rs b/crates/proc-macro-api/src/legacy_protocol.rs index ee96b899fe57..0a72052cc53b 100644 --- a/crates/proc-macro-api/src/legacy_protocol.rs +++ b/crates/proc-macro-api/src/legacy_protocol.rs @@ -95,9 +95,10 @@ pub(crate) fn expand( let mixed_site = span_data_table.insert_full(mixed_site).0; let task = ExpandMacro { data: ExpandMacroData { - macro_body: FlatTree::new(subtree, version, &mut span_data_table), + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), macro_name: proc_macro.name.to_string(), - attributes: attr.map(|subtree| FlatTree::new(subtree, version, &mut span_data_table)), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), has_global_spans: ExpnGlobals { serialize: version >= version::HAS_GLOBAL_SPANS, def_site, diff --git a/crates/proc-macro-api/src/legacy_protocol/msg.rs b/crates/proc-macro-api/src/legacy_protocol/msg.rs index b795c4558956..487f50b145e8 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg.rs @@ -297,7 +297,7 @@ mod tests { let mut span_data_table = Default::default(); let task = ExpandMacro { data: ExpandMacroData { - macro_body: FlatTree::new(tt.view(), v, &mut span_data_table), + macro_body: FlatTree::from_subtree(tt.view(), v, &mut span_data_table), macro_name: Default::default(), attributes: None, has_global_spans: ExpnGlobals { diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs index fb3542d24f46..7f19506048de 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs @@ -8,8 +8,7 @@ //! about performance here a bit. //! //! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat -//! array of numbers. See the test in the parent module to get an example -//! output. +//! array of numbers. //! //! ```json //! { @@ -35,6 +34,9 @@ //! as we don't have bincode in Cargo.toml yet, lets stick with serde_json for //! the time being. +#[cfg(feature = "sysroot-abi")] +use proc_macro_srv::TokenStream; + use std::collections::VecDeque; use intern::Symbol; @@ -120,12 +122,12 @@ struct IdentRepr { } impl FlatTree { - pub fn new( + pub fn from_subtree( subtree: tt::SubtreeView<'_, Span>, version: u32, span_data_table: &mut SpanDataIndexMap, ) -> FlatTree { - let mut w = Writer:: { + let mut w = Writer:: { string_table: FxHashMap::default(), work: VecDeque::new(), span_data_table, @@ -138,7 +140,7 @@ impl FlatTree { text: Vec::new(), version, }; - w.write(subtree); + w.write_subtree(subtree); FlatTree { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { @@ -162,11 +164,97 @@ impl FlatTree { } } - pub fn new_raw>( - subtree: tt::SubtreeView<'_, T::Span>, + pub fn to_subtree_resolved( + self, version: u32, + span_data_table: &SpanDataIndexMap, + ) -> tt::TopSubtree { + Reader:: { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + read_vec(self.subtree, SubtreeRepr::read_with_close_span) + } else { + read_vec(self.subtree, SubtreeRepr::read) + }, + literal: if version >= EXTENDED_LEAF_DATA { + read_vec(self.literal, LiteralRepr::read_with_kind) + } else { + read_vec(self.literal, LiteralRepr::read) + }, + punct: read_vec(self.punct, PunctRepr::read), + ident: if version >= EXTENDED_LEAF_DATA { + read_vec(self.ident, IdentRepr::read_with_rawness) + } else { + read_vec(self.ident, IdentRepr::read) + }, + token_tree: self.token_tree, + text: self.text, + span_data_table, + version, + } + .read_subtree() + } +} + +#[cfg(feature = "sysroot-abi")] +impl FlatTree { + pub fn from_tokenstream( + tokenstream: proc_macro_srv::TokenStream, + version: u32, + call_site: Span, + span_data_table: &mut SpanDataIndexMap, ) -> FlatTree { - let mut w = Writer:: { + let mut w = Writer:: { + string_table: FxHashMap::default(), + work: VecDeque::new(), + span_data_table, + + subtree: Vec::new(), + literal: Vec::new(), + punct: Vec::new(), + ident: Vec::new(), + token_tree: Vec::new(), + text: Vec::new(), + version, + }; + let group = proc_macro_srv::Group { + delimiter: proc_macro_srv::Delimiter::None, + stream: Some(tokenstream), + span: proc_macro_srv::DelimSpan { + open: call_site, + close: call_site, + entire: call_site, + }, + }; + w.write_tokenstream(&group); + + FlatTree { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + write_vec(w.subtree, SubtreeRepr::write_with_close_span) + } else { + write_vec(w.subtree, SubtreeRepr::write) + }, + literal: if version >= EXTENDED_LEAF_DATA { + write_vec(w.literal, LiteralRepr::write_with_kind) + } else { + write_vec(w.literal, LiteralRepr::write) + }, + punct: write_vec(w.punct, PunctRepr::write), + ident: if version >= EXTENDED_LEAF_DATA { + write_vec(w.ident, IdentRepr::write_with_rawness) + } else { + write_vec(w.ident, IdentRepr::write) + }, + token_tree: w.token_tree, + text: w.text, + } + } + + pub fn from_tokenstream_raw>( + tokenstream: proc_macro_srv::TokenStream, + call_site: T::Span, + version: u32, + ) -> FlatTree { + let mut w = Writer:: { string_table: FxHashMap::default(), work: VecDeque::new(), span_data_table: &mut (), @@ -179,7 +267,16 @@ impl FlatTree { text: Vec::new(), version, }; - w.write(subtree); + let group = proc_macro_srv::Group { + delimiter: proc_macro_srv::Delimiter::None, + stream: Some(tokenstream), + span: proc_macro_srv::DelimSpan { + open: call_site, + close: call_site, + entire: call_site, + }, + }; + w.write_tokenstream(&group); FlatTree { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { @@ -203,12 +300,11 @@ impl FlatTree { } } - pub fn to_subtree_resolved( + pub fn to_tokenstream_unresolved>( self, version: u32, - span_data_table: &SpanDataIndexMap, - ) -> tt::TopSubtree { - Reader:: { + ) -> proc_macro_srv::TokenStream { + Reader:: { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { read_vec(self.subtree, SubtreeRepr::read_with_close_span) } else { @@ -227,17 +323,18 @@ impl FlatTree { }, token_tree: self.token_tree, text: self.text, - span_data_table, + span_data_table: &(), version, } - .read() + .read_tokenstream() } - pub fn to_subtree_unresolved>( + pub fn to_tokenstream_resolved( self, version: u32, - ) -> tt::TopSubtree { - Reader:: { + span_data_table: &SpanDataIndexMap, + ) -> proc_macro_srv::TokenStream { + Reader:: { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { read_vec(self.subtree, SubtreeRepr::read_with_close_span) } else { @@ -256,10 +353,10 @@ impl FlatTree { }, token_tree: self.token_tree, text: self.text, - span_data_table: &(), + span_data_table, version, } - .read() + .read_tokenstream() } } @@ -391,8 +488,8 @@ impl SpanTransformer for Span { } } -struct Writer<'a, 'span, S: SpanTransformer> { - work: VecDeque<(usize, tt::iter::TtIter<'a, S::Span>)>, +struct Writer<'a, 'span, S: SpanTransformer, W> { + work: VecDeque<(usize, W)>, string_table: FxHashMap, u32>, span_data_table: &'span mut S::Table, version: u32, @@ -405,8 +502,8 @@ struct Writer<'a, 'span, S: SpanTransformer> { text: Vec, } -impl<'a, T: SpanTransformer> Writer<'a, '_, T> { - fn write(&mut self, root: tt::SubtreeView<'a, T::Span>) { +impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIter<'a, T::Span>> { + fn write_subtree(&mut self, root: tt::SubtreeView<'a, T::Span>) { let subtree = root.top_subtree(); self.enqueue(subtree, root.iter()); while let Some((idx, subtree)) = self.work.pop_front() { @@ -414,10 +511,6 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T> { } } - fn token_id_of(&mut self, span: T::Span) -> SpanId { - T::token_id_of(self.span_data_table, span) - } - fn subtree(&mut self, idx: usize, subtree: tt::iter::TtIter<'a, T::Span>) { let mut first_tt = self.token_tree.len(); let n_tt = subtree.clone().count(); // FIXME: `count()` walks over the entire iterator. @@ -502,6 +595,12 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T> { self.work.push_back((idx, contents)); idx as u32 } +} + +impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> { + fn token_id_of(&mut self, span: T::Span) -> SpanId { + T::token_id_of(self.span_data_table, span) + } pub(crate) fn intern(&mut self, text: &'a str) -> u32 { let table = &mut self.text; @@ -522,6 +621,105 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T> { } } +#[cfg(feature = "sysroot-abi")] +impl<'a, T: SpanTransformer> Writer<'a, '_, T, &'a proc_macro_srv::Group> { + fn write_tokenstream(&mut self, root: &'a proc_macro_srv::Group) { + self.enqueue_group(root); + + while let Some((idx, group)) = self.work.pop_front() { + self.group(idx, group); + } + } + + fn group(&mut self, idx: usize, group: &'a proc_macro_srv::Group) { + let mut first_tt = self.token_tree.len(); + let n_tt = group.stream.as_ref().map_or(0, |it| it.len()); + self.token_tree.resize(first_tt + n_tt, !0); + + self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; + + for tt in group.stream.iter().flat_map(|it| it.iter()) { + let idx_tag = match tt { + proc_macro_srv::TokenTree::Group(group) => { + let idx = self.enqueue_group(group); + idx << 2 + } + proc_macro_srv::TokenTree::Literal(lit) => { + let idx = self.literal.len() as u32; + let id = self.token_id_of(lit.span); + let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { + ( + self.intern(lit.symbol.as_str()), + lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), + ) + } else { + (self.intern_owned(proc_macro_srv::literal_to_string(lit)), !0) + }; + self.literal.push(LiteralRepr { + id, + text, + kind: u16::from_le_bytes(match lit.kind { + proc_macro_srv::LitKind::ErrWithGuar => [0, 0], + proc_macro_srv::LitKind::Byte => [1, 0], + proc_macro_srv::LitKind::Char => [2, 0], + proc_macro_srv::LitKind::Integer => [3, 0], + proc_macro_srv::LitKind::Float => [4, 0], + proc_macro_srv::LitKind::Str => [5, 0], + proc_macro_srv::LitKind::StrRaw(r) => [6, r], + proc_macro_srv::LitKind::ByteStr => [7, 0], + proc_macro_srv::LitKind::ByteStrRaw(r) => [8, r], + proc_macro_srv::LitKind::CStr => [9, 0], + proc_macro_srv::LitKind::CStrRaw(r) => [10, r], + }), + suffix, + }); + (idx << 2) | 0b01 + } + proc_macro_srv::TokenTree::Punct(punct) => { + let idx = self.punct.len() as u32; + let id = self.token_id_of(punct.span); + self.punct.push(PunctRepr { + char: punct.ch as char, + spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, + id, + }); + (idx << 2) | 0b10 + } + proc_macro_srv::TokenTree::Ident(ident) => { + let idx = self.ident.len() as u32; + let id = self.token_id_of(ident.span); + let text = if self.version >= EXTENDED_LEAF_DATA { + self.intern(ident.sym.as_str()) + } else if ident.is_raw { + self.intern_owned(format!("r#{}", ident.sym.as_str(),)) + } else { + self.intern(ident.sym.as_str()) + }; + self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw }); + (idx << 2) | 0b11 + } + }; + self.token_tree[first_tt] = idx_tag; + first_tt += 1; + } + } + + fn enqueue_group(&mut self, group: &'a proc_macro_srv::Group) -> u32 { + let idx = self.subtree.len(); + let open = self.token_id_of(group.span.open); + let close = self.token_id_of(group.span.close); + let delimiter_kind = match group.delimiter { + proc_macro_srv::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, + proc_macro_srv::Delimiter::Brace => tt::DelimiterKind::Brace, + proc_macro_srv::Delimiter::Bracket => tt::DelimiterKind::Bracket, + proc_macro_srv::Delimiter::None => tt::DelimiterKind::Invisible, + }; + self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); + self.work.push_back((idx, group)); + idx as u32 + } +} + struct Reader<'span, S: SpanTransformer> { version: u32, subtree: Vec, @@ -534,7 +732,7 @@ struct Reader<'span, S: SpanTransformer> { } impl Reader<'_, T> { - pub(crate) fn read(self) -> tt::TopSubtree { + pub(crate) fn read_subtree(self) -> tt::TopSubtree { let mut res: Vec, Vec>)>> = vec![None; self.subtree.len()]; let read_span = |id| T::span_for_token_id(self.span_data_table, id); @@ -641,3 +839,122 @@ impl Reader<'_, T> { tt::TopSubtree(res.into_boxed_slice()) } } + +#[cfg(feature = "sysroot-abi")] +impl Reader<'_, T> { + pub(crate) fn read_tokenstream(self) -> proc_macro_srv::TokenStream { + let mut res: Vec>> = vec![None; self.subtree.len()]; + let read_span = |id| T::span_for_token_id(self.span_data_table, id); + for i in (0..self.subtree.len()).rev() { + let repr = &self.subtree[i]; + let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; + + let stream = token_trees + .iter() + .copied() + .map(|idx_tag| { + let tag = idx_tag & 0b11; + let idx = (idx_tag >> 2) as usize; + match tag { + // XXX: we iterate subtrees in reverse to guarantee + // that this unwrap doesn't fire. + 0b00 => proc_macro_srv::TokenTree::Group(res[idx].take().unwrap()), + 0b01 => { + let repr = &self.literal[idx]; + let text = self.text[repr.text as usize].as_str(); + let span = read_span(repr.id); + proc_macro_srv::TokenTree::Literal( + if self.version >= EXTENDED_LEAF_DATA { + proc_macro_srv::Literal { + symbol: Symbol::intern(text), + span, + kind: match u16::to_le_bytes(repr.kind) { + [0, _] => proc_macro_srv::LitKind::ErrWithGuar, + [1, _] => proc_macro_srv::LitKind::Byte, + [2, _] => proc_macro_srv::LitKind::Char, + [3, _] => proc_macro_srv::LitKind::Integer, + [4, _] => proc_macro_srv::LitKind::Float, + [5, _] => proc_macro_srv::LitKind::Str, + [6, r] => proc_macro_srv::LitKind::StrRaw(r), + [7, _] => proc_macro_srv::LitKind::ByteStr, + [8, r] => proc_macro_srv::LitKind::ByteStrRaw(r), + [9, _] => proc_macro_srv::LitKind::CStr, + [10, r] => proc_macro_srv::LitKind::CStrRaw(r), + _ => unreachable!(), + }, + suffix: if repr.suffix != !0 { + Some(Symbol::intern( + self.text[repr.suffix as usize].as_str(), + )) + } else { + None + }, + } + } else { + proc_macro_srv::literal_from_str(text, span).unwrap_or_else( + |_| proc_macro_srv::Literal { + symbol: Symbol::intern("internal error"), + span, + kind: proc_macro_srv::LitKind::ErrWithGuar, + suffix: None, + }, + ) + }, + ) + } + 0b10 => { + let repr = &self.punct[idx]; + proc_macro_srv::TokenTree::Punct(proc_macro_srv::Punct { + ch: repr.char as u8, + joint: repr.spacing == tt::Spacing::Joint, + span: read_span(repr.id), + }) + } + 0b11 => { + let repr = &self.ident[idx]; + let text = self.text[repr.text as usize].as_str(); + let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { + ( + if repr.is_raw { + tt::IdentIsRaw::Yes + } else { + tt::IdentIsRaw::No + }, + text, + ) + } else { + tt::IdentIsRaw::split_from_symbol(text) + }; + proc_macro_srv::TokenTree::Ident(proc_macro_srv::Ident { + sym: Symbol::intern(text), + span: read_span(repr.id), + is_raw: is_raw.yes(), + }) + } + other => panic!("bad tag: {other}"), + } + }) + .collect::>(); + let g = proc_macro_srv::Group { + delimiter: match repr.kind { + tt::DelimiterKind::Parenthesis => proc_macro_srv::Delimiter::Parenthesis, + tt::DelimiterKind::Brace => proc_macro_srv::Delimiter::Brace, + tt::DelimiterKind::Bracket => proc_macro_srv::Delimiter::Bracket, + tt::DelimiterKind::Invisible => proc_macro_srv::Delimiter::None, + }, + stream: if stream.is_empty() { None } else { Some(TokenStream::new(stream)) }, + span: proc_macro_srv::DelimSpan { + open: read_span(repr.open), + close: read_span(repr.close), + // FIXME + entire: read_span(repr.close), + }, + }; + res[i] = Some(g); + } + // FIXME: double check this + proc_macro_srv::TokenStream::new(vec![proc_macro_srv::TokenTree::Group( + res[0].take().unwrap(), + )]) + } +} diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 97919b85b513..870d81f97684 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -5,6 +5,13 @@ //! is used to provide basic infrastructure for communication between two //! processes: Client (RA itself), Server (the external program) +#![cfg_attr(not(feature = "sysroot-abi"), allow(unused_crate_dependencies))] +#![cfg_attr( + feature = "sysroot-abi", + feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span) +)] +#![allow(internal_features)] + pub mod legacy_protocol; mod process; diff --git a/crates/proc-macro-srv-cli/Cargo.toml b/crates/proc-macro-srv-cli/Cargo.toml index 91e9e62b084b..dd31e74915bf 100644 --- a/crates/proc-macro-srv-cli/Cargo.toml +++ b/crates/proc-macro-srv-cli/Cargo.toml @@ -19,7 +19,7 @@ postcard = { version = "1.1.3", optional = true } [features] default = ["postcard"] -sysroot-abi = ["proc-macro-srv/sysroot-abi"] +sysroot-abi = ["proc-macro-srv/sysroot-abi", "proc-macro-api/sysroot-abi"] in-rust-tree = ["proc-macro-srv/in-rust-tree", "sysroot-abi"] postcard = ["dep:postcard"] diff --git a/crates/proc-macro-srv-cli/build.rs b/crates/proc-macro-srv-cli/build.rs index 12e7c8b05bac..bcf639fefca7 100644 --- a/crates/proc-macro-srv-cli/build.rs +++ b/crates/proc-macro-srv-cli/build.rs @@ -5,7 +5,6 @@ use std::{env, path::PathBuf, process::Command}; fn main() { set_rerun(); set_commit_info(); - println!("cargo::rustc-check-cfg=cfg(rust_analyzer)"); } fn set_rerun() { diff --git a/crates/proc-macro-srv-cli/src/main.rs b/crates/proc-macro-srv-cli/src/main.rs index 662d34865eff..9d74fa637aa9 100644 --- a/crates/proc-macro-srv-cli/src/main.rs +++ b/crates/proc-macro-srv-cli/src/main.rs @@ -9,10 +9,10 @@ extern crate rustc_driver as _; mod version; -#[cfg(any(feature = "sysroot-abi", rust_analyzer))] +#[cfg(feature = "sysroot-abi")] mod main_loop; use clap::{Command, ValueEnum}; -#[cfg(any(feature = "sysroot-abi", rust_analyzer))] +#[cfg(feature = "sysroot-abi")] use main_loop::run; fn main() -> std::io::Result<()> { @@ -77,7 +77,7 @@ impl ValueEnum for ProtocolFormat { } } -#[cfg(not(any(feature = "sysroot-abi", rust_analyzer)))] +#[cfg(not(feature = "sysroot-abi"))] fn run(_: ProtocolFormat) -> std::io::Result<()> { Err(std::io::Error::new( std::io::ErrorKind::Unsupported, diff --git a/crates/proc-macro-srv-cli/src/main_loop.rs b/crates/proc-macro-srv-cli/src/main_loop.rs index 703bc965db25..55331075704a 100644 --- a/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/crates/proc-macro-srv-cli/src/main_loop.rs @@ -91,9 +91,10 @@ fn run_json() -> io::Result<()> { let mixed_site = SpanId(mixed_site as u32); let macro_body = - macro_body.to_subtree_unresolved::(CURRENT_API_VERSION); - let attributes = attributes - .map(|it| it.to_subtree_unresolved::(CURRENT_API_VERSION)); + macro_body.to_tokenstream_unresolved::(CURRENT_API_VERSION); + let attributes = attributes.map(|it| { + it.to_tokenstream_unresolved::(CURRENT_API_VERSION) + }); srv.expand( lib, @@ -107,8 +108,9 @@ fn run_json() -> io::Result<()> { mixed_site, ) .map(|it| { - msg::FlatTree::new_raw::( - tt::SubtreeView::new(&it), + msg::FlatTree::from_tokenstream_raw::( + it, + call_site, CURRENT_API_VERSION, ) }) @@ -122,10 +124,10 @@ fn run_json() -> io::Result<()> { let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = - macro_body.to_subtree_resolved(CURRENT_API_VERSION, &span_data_table); + let macro_body = macro_body + .to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table); let attributes = attributes.map(|it| { - it.to_subtree_resolved(CURRENT_API_VERSION, &span_data_table) + it.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table) }); srv.expand( lib, @@ -140,9 +142,10 @@ fn run_json() -> io::Result<()> { ) .map(|it| { ( - msg::FlatTree::new( - tt::SubtreeView::new(&it), + msg::FlatTree::from_tokenstream( + it, CURRENT_API_VERSION, + call_site, &mut span_data_table, ), serialize_span_data_index_map(&span_data_table), diff --git a/crates/proc-macro-srv/Cargo.toml b/crates/proc-macro-srv/Cargo.toml index d037e715e703..361017178409 100644 --- a/crates/proc-macro-srv/Cargo.toml +++ b/crates/proc-macro-srv/Cargo.toml @@ -18,8 +18,6 @@ libloading.workspace = true memmap2.workspace = true temp-dir.workspace = true -tt.workspace = true -syntax-bridge.workspace = true paths.workspace = true # span = {workspace = true, default-features = false} does not work span = { path = "../span", version = "0.0.0", default-features = false} @@ -38,8 +36,9 @@ expect-test.workspace = true proc-macro-test.path = "./proc-macro-test" [features] +default = [] sysroot-abi = [] -in-rust-tree = ["syntax-bridge/in-rust-tree", "tt/in-rust-tree", "sysroot-abi"] +in-rust-tree = ["sysroot-abi"] [lints] workspace = true diff --git a/crates/proc-macro-srv/build.rs b/crates/proc-macro-srv/build.rs index 97c0c4bda7df..4cf1820e7ba9 100644 --- a/crates/proc-macro-srv/build.rs +++ b/crates/proc-macro-srv/build.rs @@ -4,8 +4,6 @@ use std::{env, process::Command}; fn main() { - println!("cargo::rustc-check-cfg=cfg(rust_analyzer)"); - let rustc = env::var("RUSTC").expect("proc-macro-srv's build script expects RUSTC to be set"); #[allow(clippy::disallowed_methods)] let output = Command::new(rustc).arg("--version").output().expect("rustc --version must run"); diff --git a/crates/proc-macro-srv/src/bridge.rs b/crates/proc-macro-srv/src/bridge.rs new file mode 100644 index 000000000000..fc063a07b5f8 --- /dev/null +++ b/crates/proc-macro-srv/src/bridge.rs @@ -0,0 +1,12 @@ +//! `proc_macro::bridge` newtypes. + +use proc_macro::bridge as pm_bridge; + +pub use pm_bridge::{DelimSpan, Diagnostic, ExpnGlobals, LitKind}; + +pub type TokenTree = + pm_bridge::TokenTree, S, intern::Symbol>; +pub type Literal = pm_bridge::Literal; +pub type Group = pm_bridge::Group, S>; +pub type Punct = pm_bridge::Punct; +pub type Ident = pm_bridge::Ident; diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index c8513a10675d..03433197b779 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -13,7 +13,7 @@ use paths::{Utf8Path, Utf8PathBuf}; use crate::{ PanicMessage, ProcMacroKind, ProcMacroSrvSpan, dylib::proc_macros::ProcMacros, - server_impl::TopSubtree, + token_stream::TokenStream, }; pub(crate) struct Expander { @@ -40,18 +40,18 @@ impl Expander { pub(crate) fn expand( &self, macro_name: &str, - macro_body: TopSubtree, - attributes: Option>, + macro_body: TokenStream, + attribute: Option>, def_site: S, call_site: S, mixed_site: S, - ) -> Result, PanicMessage> + ) -> Result, PanicMessage> where ::TokenStream: Default, { self.inner .proc_macros - .expand(macro_name, macro_body, attributes, def_site, call_site, mixed_site) + .expand(macro_name, macro_body, attribute, def_site, call_site, mixed_site) } pub(crate) fn list_macros(&self) -> impl Iterator { diff --git a/crates/proc-macro-srv/src/dylib/proc_macros.rs b/crates/proc-macro-srv/src/dylib/proc_macros.rs index 9b5721e370ac..c879c7609d91 100644 --- a/crates/proc-macro-srv/src/dylib/proc_macros.rs +++ b/crates/proc-macro-srv/src/dylib/proc_macros.rs @@ -2,7 +2,7 @@ use proc_macro::bridge; -use crate::{ProcMacroKind, ProcMacroSrvSpan, server_impl::TopSubtree}; +use crate::{ProcMacroKind, ProcMacroSrvSpan, token_stream::TokenStream}; #[repr(transparent)] pub(crate) struct ProcMacros([bridge::client::ProcMacro]); @@ -17,18 +17,13 @@ impl ProcMacros { pub(crate) fn expand( &self, macro_name: &str, - macro_body: TopSubtree, - attributes: Option>, + macro_body: TokenStream, + attribute: Option>, def_site: S, call_site: S, mixed_site: S, - ) -> Result, crate::PanicMessage> { - let parsed_body = crate::server_impl::TokenStream::with_subtree(macro_body); - - let parsed_attributes = attributes - .map_or_else(crate::server_impl::TokenStream::default, |attr| { - crate::server_impl::TokenStream::with_subtree(attr) - }); + ) -> Result, crate::PanicMessage> { + let parsed_attributes = attribute.unwrap_or_default(); for proc_macro in &self.0 { match proc_macro { @@ -38,35 +33,29 @@ impl ProcMacros { let res = client.run( &bridge::server::SameThread, S::make_server(call_site, def_site, mixed_site), - parsed_body, + macro_body, cfg!(debug_assertions), ); - return res - .map(|it| it.into_subtree(call_site)) - .map_err(crate::PanicMessage::from); + return res.map_err(crate::PanicMessage::from); } bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => { let res = client.run( &bridge::server::SameThread, S::make_server(call_site, def_site, mixed_site), - parsed_body, + macro_body, cfg!(debug_assertions), ); - return res - .map(|it| it.into_subtree(call_site)) - .map_err(crate::PanicMessage::from); + return res.map_err(crate::PanicMessage::from); } bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => { let res = client.run( &bridge::server::SameThread, S::make_server(call_site, def_site, mixed_site), parsed_attributes, - parsed_body, + macro_body, cfg!(debug_assertions), ); - return res - .map(|it| it.into_subtree(call_site)) - .map_err(crate::PanicMessage::from); + return res.map_err(crate::PanicMessage::from); } _ => continue, } diff --git a/crates/proc-macro-srv/src/dylib/version.rs b/crates/proc-macro-srv/src/dylib/version.rs index 3b2551f08c48..209693b4daec 100644 --- a/crates/proc-macro-srv/src/dylib/version.rs +++ b/crates/proc-macro-srv/src/dylib/version.rs @@ -5,11 +5,14 @@ use std::io::{self, Read}; use object::read::{Object, ObjectSection}; #[derive(Debug)] -#[allow(dead_code)] pub struct RustCInfo { + #[allow(dead_code)] pub version: (usize, usize, usize), + #[allow(dead_code)] pub channel: String, + #[allow(dead_code)] pub commit: Option, + #[allow(dead_code)] pub date: Option, // something like "rustc 1.58.1 (db9d1b20b 2022-01-20)" pub version_string: String, diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index cb97882c5854..aff4dc50378c 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -10,11 +10,16 @@ //! * By **copying** the whole rustc `lib_proc_macro` code, we are able to build this with `stable` //! rustc rather than `unstable`. (Although in general ABI compatibility is still an issue)… -#![cfg(any(feature = "sysroot-abi", rust_analyzer))] -#![cfg_attr(not(feature = "sysroot-abi"), allow(unused_crate_dependencies))] +#![cfg(feature = "sysroot-abi")] #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))] #![feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span)] -#![allow(unreachable_pub, internal_features, clippy::disallowed_types, clippy::print_stderr)] +#![allow( + unreachable_pub, + internal_features, + clippy::disallowed_types, + clippy::print_stderr, + unused_crate_dependencies +)] #![deny(deprecated_safe, clippy::undocumented_unsafe_blocks)] extern crate proc_macro; @@ -26,8 +31,10 @@ extern crate ra_ap_rustc_lexer as rustc_lexer; #[cfg(feature = "in-rust-tree")] extern crate rustc_lexer; +mod bridge; mod dylib; mod server_impl; +mod token_stream; use std::{ collections::{HashMap, hash_map::Entry}, @@ -43,10 +50,14 @@ use paths::{Utf8Path, Utf8PathBuf}; use span::Span; use temp_dir::TempDir; -use crate::server_impl::TokenStream; - pub use crate::server_impl::token_id::SpanId; +pub use proc_macro::Delimiter; + +pub use crate::bridge::*; +pub use crate::server_impl::literal_from_str; +pub use crate::token_stream::{TokenStream, literal_to_string}; + #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ProcMacroKind { CustomDerive, @@ -81,12 +92,12 @@ impl ProcMacroSrv<'_> { env: &[(String, String)], current_dir: Option>, macro_name: &str, - macro_body: tt::TopSubtree, - attribute: Option>, + macro_body: token_stream::TokenStream, + attribute: Option>, def_site: S, call_site: S, mixed_site: S, - ) -> Result>, PanicMessage> { + ) -> Result, PanicMessage> { let snapped_env = self.env; let expander = self.expander(lib.as_ref()).map_err(|err| PanicMessage { message: Some(format!("failed to load macro: {err}")), @@ -102,15 +113,7 @@ impl ProcMacroSrv<'_> { .name(macro_name.to_owned()) .spawn_scoped(s, move || { expander - .expand( - macro_name, - server_impl::TopSubtree(macro_body.0.into_vec()), - attribute.map(|it| server_impl::TopSubtree(it.0.into_vec())), - def_site, - call_site, - mixed_site, - ) - .map(|tt| tt.0) + .expand(macro_name, macro_body, attribute, def_site, call_site, mixed_site) }); match thread.unwrap().join() { Ok(res) => res, @@ -157,8 +160,8 @@ impl ProcMacroSrv<'_> { } } -pub trait ProcMacroSrvSpan: Copy + Send { - type Server: proc_macro::bridge::server::Server>; +pub trait ProcMacroSrvSpan: Copy + Send + Sync { + type Server: proc_macro::bridge::server::Server>; fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server; } diff --git a/crates/proc-macro-srv/src/server_impl.rs b/crates/proc-macro-srv/src/server_impl.rs index 32ad32731ba6..bacead1a88da 100644 --- a/crates/proc-macro-srv/src/server_impl.rs +++ b/crates/proc-macro-srv/src/server_impl.rs @@ -5,122 +5,15 @@ //! we could provide any TokenStream implementation. //! The original idea from fedochet is using proc-macro2 as backend, //! we use tt instead for better integration with RA. -//! -//! FIXME: No span and source file information is implemented yet - -use std::fmt; - -use intern::Symbol; -use proc_macro::bridge; - -mod token_stream; -pub use token_stream::TokenStream; - -pub mod rust_analyzer_span; -pub mod token_id; - -use tt::Spacing; - -#[derive(Clone)] -pub(crate) struct TopSubtree(pub(crate) Vec>); - -impl fmt::Debug for TopSubtree { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&tt::TokenTreesView::new(&self.0), f) - } -} - -impl TopSubtree { - pub(crate) fn top_subtree(&self) -> &tt::Subtree { - let tt::TokenTree::Subtree(subtree) = &self.0[0] else { - unreachable!("the first token tree is always the top subtree"); - }; - subtree - } - pub(crate) fn from_bridge(group: bridge::Group, S>) -> Self { - let delimiter = delim_to_internal(group.delimiter, group.span); - let mut tts = - group.stream.map(|it| it.token_trees).unwrap_or_else(|| Vec::with_capacity(1)); - tts.insert(0, tt::TokenTree::Subtree(tt::Subtree { delimiter, len: tts.len() as u32 })); - TopSubtree(tts) - } -} - -fn delim_to_internal(d: proc_macro::Delimiter, span: bridge::DelimSpan) -> tt::Delimiter { - let kind = match d { - proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, - proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace, - proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket, - proc_macro::Delimiter::None => tt::DelimiterKind::Invisible, - }; - tt::Delimiter { open: span.open, close: span.close, kind } -} - -fn delim_to_external(d: tt::Delimiter) -> proc_macro::Delimiter { - match d.kind { - tt::DelimiterKind::Parenthesis => proc_macro::Delimiter::Parenthesis, - tt::DelimiterKind::Brace => proc_macro::Delimiter::Brace, - tt::DelimiterKind::Bracket => proc_macro::Delimiter::Bracket, - tt::DelimiterKind::Invisible => proc_macro::Delimiter::None, - } -} - -#[allow(unused)] -fn spacing_to_internal(spacing: proc_macro::Spacing) -> Spacing { - match spacing { - proc_macro::Spacing::Alone => Spacing::Alone, - proc_macro::Spacing::Joint => Spacing::Joint, - } -} +pub(crate) mod rust_analyzer_span; +pub(crate) mod token_id; -#[allow(unused)] -fn spacing_to_external(spacing: Spacing) -> proc_macro::Spacing { - match spacing { - Spacing::Alone | Spacing::JointHidden => proc_macro::Spacing::Alone, - Spacing::Joint => proc_macro::Spacing::Joint, - } -} - -fn literal_kind_to_external(kind: tt::LitKind) -> bridge::LitKind { - match kind { - tt::LitKind::Byte => bridge::LitKind::Byte, - tt::LitKind::Char => bridge::LitKind::Char, - tt::LitKind::Integer => bridge::LitKind::Integer, - tt::LitKind::Float => bridge::LitKind::Float, - tt::LitKind::Str => bridge::LitKind::Str, - tt::LitKind::StrRaw(r) => bridge::LitKind::StrRaw(r), - tt::LitKind::ByteStr => bridge::LitKind::ByteStr, - tt::LitKind::ByteStrRaw(r) => bridge::LitKind::ByteStrRaw(r), - tt::LitKind::CStr => bridge::LitKind::CStr, - tt::LitKind::CStrRaw(r) => bridge::LitKind::CStrRaw(r), - tt::LitKind::Err(_) => bridge::LitKind::ErrWithGuar, - } -} - -fn literal_kind_to_internal(kind: bridge::LitKind) -> tt::LitKind { - match kind { - bridge::LitKind::Byte => tt::LitKind::Byte, - bridge::LitKind::Char => tt::LitKind::Char, - bridge::LitKind::Str => tt::LitKind::Str, - bridge::LitKind::StrRaw(r) => tt::LitKind::StrRaw(r), - bridge::LitKind::ByteStr => tt::LitKind::ByteStr, - bridge::LitKind::ByteStrRaw(r) => tt::LitKind::ByteStrRaw(r), - bridge::LitKind::CStr => tt::LitKind::CStr, - bridge::LitKind::CStrRaw(r) => tt::LitKind::CStrRaw(r), - bridge::LitKind::Integer => tt::LitKind::Integer, - bridge::LitKind::Float => tt::LitKind::Float, - bridge::LitKind::ErrWithGuar => tt::LitKind::Err(()), - } -} - -pub(super) fn literal_from_str( +pub fn literal_from_str( s: &str, span: Span, -) -> Result, ()> { - use proc_macro::bridge::LitKind; +) -> Result, ()> { use rustc_lexer::{LiteralKind, Token, TokenKind}; - let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No); let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 }); @@ -142,98 +35,5 @@ pub(super) fn literal_from_str( } let TokenKind::Literal { kind, suffix_start } = lit.kind else { return Err(()) }; - let (kind, start_offset, end_offset) = match kind { - LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), - LiteralKind::Float { .. } => (LitKind::Float, 0, 0), - LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), - LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), - LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), - LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), - LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), - LiteralKind::RawStr { n_hashes } => ( - LitKind::StrRaw(n_hashes.unwrap_or_default()), - 2 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawByteStr { n_hashes } => ( - LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - LiteralKind::RawCStr { n_hashes } => ( - LitKind::CStrRaw(n_hashes.unwrap_or_default()), - 3 + n_hashes.unwrap_or_default() as usize, - 1 + n_hashes.unwrap_or_default() as usize, - ), - }; - - let (lit, suffix) = s.split_at(suffix_start as usize); - let lit = &lit[start_offset..lit.len() - end_offset]; - let suffix = match suffix { - "" | "_" => None, - suffix => Some(Symbol::intern(suffix)), - }; - - Ok(bridge::Literal { kind, symbol: Symbol::intern(lit), suffix, span }) -} - -pub(super) fn from_token_tree( - tree: bridge::TokenTree, Span, Symbol>, -) -> TokenStream { - match tree { - bridge::TokenTree::Group(group) => { - let group = TopSubtree::from_bridge(group); - TokenStream { token_trees: group.0 } - } - - bridge::TokenTree::Ident(ident) => { - let text = ident.sym; - let ident: tt::Ident = tt::Ident { - sym: text, - span: ident.span, - is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }, - }; - let leaf = tt::Leaf::from(ident); - let tree = tt::TokenTree::from(leaf); - TokenStream { token_trees: vec![tree] } - } - - bridge::TokenTree::Literal(literal) => { - let mut token_trees = Vec::new(); - let mut symbol = literal.symbol; - if matches!( - literal.kind, - proc_macro::bridge::LitKind::Integer | proc_macro::bridge::LitKind::Float - ) && symbol.as_str().starts_with('-') - { - token_trees.push(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { - spacing: tt::Spacing::Alone, - span: literal.span, - char: '-', - }))); - symbol = Symbol::intern(&symbol.as_str()[1..]); - } - let literal = tt::Literal { - symbol, - suffix: literal.suffix, - span: literal.span, - kind: literal_kind_to_internal(literal.kind), - }; - let leaf: tt::Leaf = tt::Leaf::from(literal); - let tree = tt::TokenTree::from(leaf); - token_trees.push(tree); - TokenStream { token_trees } - } - - bridge::TokenTree::Punct(p) => { - let punct = tt::Punct { - char: p.ch as char, - spacing: if p.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, - span: p.span, - }; - let leaf = tt::Leaf::from(punct); - let tree = tt::TokenTree::from(leaf); - TokenStream { token_trees: vec![tree] } - } - } + Ok(crate::token_stream::literal_from_lexer(s, span, kind, suffix_start)) } diff --git a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index a1863efafbb7..7c685c2da734 100644 --- a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -10,13 +10,13 @@ use std::{ }; use intern::Symbol; -use proc_macro::bridge::{self, server}; -use span::{FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; -use tt::{TextRange, TextSize}; +use proc_macro::bridge::server; +use span::{FIXUP_ERASED_FILE_AST_ID_MARKER, Span, TextRange, TextSize}; -use crate::server_impl::{from_token_tree, literal_from_str, token_stream::TokenStreamBuilder}; - -type TokenStream = crate::server_impl::TokenStream; +use crate::{ + bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, + server_impl::literal_from_str, +}; pub struct FreeFunctions; @@ -32,7 +32,7 @@ pub struct RaSpanServer { impl server::Types for RaSpanServer { type FreeFunctions = FreeFunctions; - type TokenStream = TokenStream; + type TokenStream = crate::token_stream::TokenStream; type Span = Span; type Symbol = Symbol; } @@ -49,14 +49,11 @@ impl server::FreeFunctions for RaSpanServer { self.tracked_paths.insert(path.into()); } - fn literal_from_str( - &mut self, - s: &str, - ) -> Result, ()> { + fn literal_from_str(&mut self, s: &str) -> Result, ()> { literal_from_str(s, self.call_site) } - fn emit_diagnostic(&mut self, _: bridge::Diagnostic) { + fn emit_diagnostic(&mut self, _: Diagnostic) { // FIXME handle diagnostic } } @@ -77,11 +74,9 @@ impl server::TokenStream for RaSpanServer { fn to_string(&mut self, stream: &Self::TokenStream) -> String { stream.to_string() } - fn from_token_tree( - &mut self, - tree: bridge::TokenTree, - ) -> Self::TokenStream { - from_token_tree(tree) + + fn from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { + Self::TokenStream::new(vec![tree]) } fn expand_expr(&mut self, self_: &Self::TokenStream) -> Result { @@ -96,16 +91,17 @@ impl server::TokenStream for RaSpanServer { fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { - let mut builder = TokenStreamBuilder::default(); - if let Some(base) = base { - builder.push(base); - } - for tree in trees { - builder.push(self.from_token_tree(tree)); + match base { + Some(mut base) => { + for tt in trees { + base.push_tree(tt); + } + base + } + None => Self::TokenStream::new(trees), } - builder.build() } fn concat_streams( @@ -113,23 +109,15 @@ impl server::TokenStream for RaSpanServer { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut builder = TokenStreamBuilder::default(); - if let Some(base) = base { - builder.push(base); - } - for stream in streams { - builder.push(stream); + let mut stream = base.unwrap_or_default(); + for s in streams { + stream.push_stream(s); } - builder.build() + stream } - fn into_trees( - &mut self, - stream: Self::TokenStream, - ) -> Vec> { - stream.into_bridge(&mut |first, second| { - server::Span::join(self, first, second).unwrap_or(first) - }) + fn into_trees(&mut self, stream: Self::TokenStream) -> Vec> { + (*stream.0).clone() } } @@ -289,8 +277,8 @@ impl server::Symbol for RaSpanServer { } impl server::Server for RaSpanServer { - fn globals(&mut self) -> bridge::ExpnGlobals { - bridge::ExpnGlobals { + fn globals(&mut self) -> ExpnGlobals { + ExpnGlobals { def_site: self.def_site, call_site: self.call_site, mixed_site: self.mixed_site, @@ -305,97 +293,3 @@ impl server::Server for RaSpanServer { f(symbol.as_str()) } } - -#[cfg(test)] -mod tests { - use span::{EditionedFileId, FileId, SyntaxContext}; - - use super::*; - - #[test] - fn test_ra_server_to_string() { - let span = Span { - range: TextRange::empty(TextSize::new(0)), - anchor: span::SpanAnchor { - file_id: EditionedFileId::current_edition(FileId::from_raw(0)), - ast_id: span::ROOT_ERASED_FILE_AST_ID, - }, - ctx: SyntaxContext::root(span::Edition::CURRENT), - }; - let s = TokenStream { - token_trees: vec![ - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern("struct"), - span, - is_raw: tt::IdentIsRaw::No, - })), - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern("T"), - span, - is_raw: tt::IdentIsRaw::No, - })), - tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter { - open: span, - close: span, - kind: tt::DelimiterKind::Brace, - }, - len: 1, - }), - tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - kind: tt::LitKind::Str, - symbol: Symbol::intern("string"), - suffix: None, - span, - })), - ], - }; - - assert_eq!(s.to_string(), "struct T {\"string\"}"); - } - - #[test] - fn test_ra_server_from_str() { - let span = Span { - range: TextRange::empty(TextSize::new(0)), - anchor: span::SpanAnchor { - file_id: EditionedFileId::current_edition(FileId::from_raw(0)), - ast_id: span::ROOT_ERASED_FILE_AST_ID, - }, - ctx: SyntaxContext::root(span::Edition::CURRENT), - }; - let subtree_paren_a = vec![ - tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter { - open: span, - close: span, - kind: tt::DelimiterKind::Parenthesis, - }, - len: 1, - }), - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - is_raw: tt::IdentIsRaw::No, - sym: Symbol::intern("a"), - span, - })), - ]; - - let t1 = TokenStream::from_str("(a)", span).unwrap(); - assert_eq!(t1.token_trees.len(), 2); - assert!(t1.token_trees == subtree_paren_a); - - let t2 = TokenStream::from_str("(a);", span).unwrap(); - assert_eq!(t2.token_trees.len(), 3); - assert!(t2.token_trees[0..2] == subtree_paren_a); - - let underscore = TokenStream::from_str("_", span).unwrap(); - assert!( - underscore.token_trees[0] - == tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern("_"), - span, - is_raw: tt::IdentIsRaw::No, - })) - ); - } -} diff --git a/crates/proc-macro-srv/src/server_impl/token_id.rs b/crates/proc-macro-srv/src/server_impl/token_id.rs index 91e70ea243ae..3814320cbe96 100644 --- a/crates/proc-macro-srv/src/server_impl/token_id.rs +++ b/crates/proc-macro-srv/src/server_impl/token_id.rs @@ -3,9 +3,12 @@ use std::ops::{Bound, Range}; use intern::Symbol; -use proc_macro::bridge::{self, server}; +use proc_macro::bridge::server; -use crate::server_impl::{from_token_tree, literal_from_str, token_stream::TokenStreamBuilder}; +use crate::{ + bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, + server_impl::literal_from_str, +}; #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct SpanId(pub u32); @@ -17,7 +20,6 @@ impl std::fmt::Debug for SpanId { } type Span = SpanId; -type TokenStream = crate::server_impl::TokenStream; pub struct FreeFunctions; @@ -29,7 +31,7 @@ pub struct SpanIdServer { impl server::Types for SpanIdServer { type FreeFunctions = FreeFunctions; - type TokenStream = TokenStream; + type TokenStream = crate::token_stream::TokenStream; type Span = Span; type Symbol = Symbol; } @@ -40,14 +42,11 @@ impl server::FreeFunctions for SpanIdServer { } fn track_env_var(&mut self, _var: &str, _value: Option<&str>) {} fn track_path(&mut self, _path: &str) {} - fn literal_from_str( - &mut self, - s: &str, - ) -> Result, ()> { + fn literal_from_str(&mut self, s: &str) -> Result, ()> { literal_from_str(s, self.call_site) } - fn emit_diagnostic(&mut self, _: bridge::Diagnostic) {} + fn emit_diagnostic(&mut self, _: Diagnostic) {} } impl server::TokenStream for SpanIdServer { @@ -66,11 +65,8 @@ impl server::TokenStream for SpanIdServer { fn to_string(&mut self, stream: &Self::TokenStream) -> String { stream.to_string() } - fn from_token_tree( - &mut self, - tree: bridge::TokenTree, - ) -> Self::TokenStream { - from_token_tree(tree) + fn from_token_tree(&mut self, tree: TokenTree) -> Self::TokenStream { + Self::TokenStream::new(vec![tree]) } fn expand_expr(&mut self, self_: &Self::TokenStream) -> Result { @@ -80,16 +76,17 @@ impl server::TokenStream for SpanIdServer { fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { - let mut builder = TokenStreamBuilder::default(); - if let Some(base) = base { - builder.push(base); - } - for tree in trees { - builder.push(self.from_token_tree(tree)); + match base { + Some(mut base) => { + for tt in trees { + base.push_tree(tt); + } + base + } + None => Self::TokenStream::new(trees), } - builder.build() } fn concat_streams( @@ -97,22 +94,15 @@ impl server::TokenStream for SpanIdServer { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut builder = TokenStreamBuilder::default(); - if let Some(base) = base { - builder.push(base); + let mut stream = base.unwrap_or_default(); + for s in streams { + stream.push_stream(s); } - for stream in streams { - builder.push(stream); - } - builder.build() + stream } - fn into_trees( - &mut self, - stream: Self::TokenStream, - ) -> Vec> { - // Can't join with `SpanId`. - stream.into_bridge(&mut |first, _second| first) + fn into_trees(&mut self, stream: Self::TokenStream) -> Vec> { + (*stream.0).clone() } } @@ -191,8 +181,8 @@ impl server::Symbol for SpanIdServer { } impl server::Server for SpanIdServer { - fn globals(&mut self) -> bridge::ExpnGlobals { - bridge::ExpnGlobals { + fn globals(&mut self) -> ExpnGlobals { + ExpnGlobals { def_site: self.def_site, call_site: self.call_site, mixed_site: self.mixed_site, @@ -207,73 +197,3 @@ impl server::Server for SpanIdServer { f(symbol.as_str()) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_ra_server_to_string() { - let s = TokenStream { - token_trees: vec![ - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern("struct"), - span: SpanId(0), - is_raw: tt::IdentIsRaw::No, - })), - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern("T"), - span: SpanId(0), - is_raw: tt::IdentIsRaw::No, - })), - tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter { - open: SpanId(0), - close: SpanId(0), - kind: tt::DelimiterKind::Brace, - }, - len: 0, - }), - ], - }; - - assert_eq!(s.to_string(), "struct T {}"); - } - - #[test] - fn test_ra_server_from_str() { - let subtree_paren_a = vec![ - tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter { - open: SpanId(0), - close: SpanId(0), - kind: tt::DelimiterKind::Parenthesis, - }, - len: 1, - }), - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - is_raw: tt::IdentIsRaw::No, - sym: Symbol::intern("a"), - span: SpanId(0), - })), - ]; - - let t1 = TokenStream::from_str("(a)", SpanId(0)).unwrap(); - assert_eq!(t1.token_trees.len(), 2); - assert!(t1.token_trees[0..2] == subtree_paren_a); - - let t2 = TokenStream::from_str("(a);", SpanId(0)).unwrap(); - assert_eq!(t2.token_trees.len(), 3); - assert!(t2.token_trees[0..2] == subtree_paren_a); - - let underscore = TokenStream::from_str("_", SpanId(0)).unwrap(); - assert!( - underscore.token_trees[0] - == tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - sym: Symbol::intern("_"), - span: SpanId(0), - is_raw: tt::IdentIsRaw::No, - })) - ); - } -} diff --git a/crates/proc-macro-srv/src/server_impl/token_stream.rs b/crates/proc-macro-srv/src/server_impl/token_stream.rs deleted file mode 100644 index c5019a591722..000000000000 --- a/crates/proc-macro-srv/src/server_impl/token_stream.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! TokenStream implementation used by sysroot ABI - -use proc_macro::bridge; - -use crate::server_impl::{TopSubtree, delim_to_external, literal_kind_to_external}; - -#[derive(Clone)] -pub struct TokenStream { - pub(super) token_trees: Vec>, -} - -// #[derive(Default)] would mean that `S: Default`. -impl Default for TokenStream { - fn default() -> Self { - Self { token_trees: Default::default() } - } -} - -impl std::fmt::Debug for TokenStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TokenStream") - .field("token_trees", &tt::TokenTreesView::new(&self.token_trees)) - .finish() - } -} - -impl TokenStream { - pub(crate) fn with_subtree(subtree: TopSubtree) -> Self { - let delimiter_kind = subtree.top_subtree().delimiter.kind; - let mut token_trees = subtree.0; - if delimiter_kind == tt::DelimiterKind::Invisible { - token_trees.remove(0); - } - TokenStream { token_trees } - } - - pub(crate) fn into_subtree(mut self, call_site: S) -> TopSubtree - where - S: Copy, - { - self.token_trees.insert( - 0, - tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter { - open: call_site, - close: call_site, - kind: tt::DelimiterKind::Invisible, - }, - len: self.token_trees.len() as u32, - }), - ); - TopSubtree(self.token_trees) - } - - pub(super) fn is_empty(&self) -> bool { - self.token_trees.is_empty() - } - - pub(crate) fn into_bridge( - self, - join_spans: &mut dyn FnMut(S, S) -> S, - ) -> Vec> { - let mut result = Vec::new(); - let mut iter = self.token_trees.into_iter(); - while let Some(tree) = iter.next() { - match tree { - tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { - result.push(bridge::TokenTree::Ident(bridge::Ident { - sym: ident.sym, - is_raw: ident.is_raw.yes(), - span: ident.span, - })) - } - // Note, we do not have to assemble our `-` punct and literal split into a single - // negative bridge literal here. As the proc-macro docs state - // > Literals created from negative numbers might not survive round-trips through - // > TokenStream or strings and may be broken into two tokens (- and positive - // > literal). - tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - result.push(bridge::TokenTree::Literal(bridge::Literal { - span: lit.span, - kind: literal_kind_to_external(lit.kind), - symbol: lit.symbol, - suffix: lit.suffix, - })) - } - tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { - result.push(bridge::TokenTree::Punct(bridge::Punct { - ch: punct.char as u8, - joint: punct.spacing == tt::Spacing::Joint, - span: punct.span, - })) - } - tt::TokenTree::Subtree(subtree) => { - result.push(bridge::TokenTree::Group(bridge::Group { - delimiter: delim_to_external(subtree.delimiter), - stream: if subtree.len == 0 { - None - } else { - Some(TokenStream { - token_trees: iter.by_ref().take(subtree.usize_len()).collect(), - }) - }, - span: bridge::DelimSpan { - open: subtree.delimiter.open, - close: subtree.delimiter.close, - entire: join_spans(subtree.delimiter.open, subtree.delimiter.close), - }, - })) - } - } - } - result - } -} - -pub(super) struct TokenStreamBuilder { - acc: TokenStream, -} - -/// pub(super)lic implementation details for the `TokenStream` type, such as iterators. -pub(super) mod token_stream_impls { - - use core::fmt; - - use super::{TokenStream, TopSubtree}; - - /// Attempts to break the string into tokens and parse those tokens into a token stream. - /// May fail for a number of reasons, for example, if the string contains unbalanced delimiters - /// or characters not existing in the language. - /// All tokens in the parsed stream get `Span::call_site()` spans. - /// - /// NOTE: some errors may cause panics instead of returning `LexError`. We reserve the right to - /// change these errors into `LexError`s later. - impl TokenStream { - pub(crate) fn from_str(src: &str, call_site: S) -> Result, String> { - let subtree = syntax_bridge::parse_to_token_tree_static_span( - span::Edition::CURRENT_FIXME, - call_site, - src, - ) - .ok_or_else(|| format!("lexing error: {src}"))?; - - Ok(TokenStream::with_subtree(TopSubtree(subtree.0.into_vec()))) - } - } - - #[allow(clippy::to_string_trait_impl)] - impl ToString for TokenStream { - fn to_string(&self) -> String { - ::tt::pretty(&self.token_trees) - } - } -} - -impl TokenStreamBuilder { - pub(super) fn push(&mut self, stream: TokenStream) { - self.acc.token_trees.extend(stream.token_trees) - } - - pub(super) fn build(self) -> TokenStream { - self.acc - } -} - -impl Default for TokenStreamBuilder { - fn default() -> Self { - Self { acc: TokenStream::default() } - } -} diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index d4f9976c92bd..1e2e8da60cde 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -10,25 +10,45 @@ use expect_test::expect; fn test_derive_empty() { assert_expand( "DeriveEmpty", - r#"struct S;"#, + r#"struct S { field: &'r#lt fn(u32) -> &'a r#u32 }"#, expect![[r#" - SUBTREE $$ 1 1 - IDENT struct 1 - IDENT S 1 - PUNCH ; [alone] 1 - - - - SUBTREE $$ 1 1"#]], + IDENT 1 struct + IDENT 1 S + GROUP {} 1 1 1 + IDENT 1 field + PUNCT 1 : [alone] + PUNCT 1 & [joint] + PUNCT 1 ' [joint] + IDENT 1 r#lt + IDENT 1 fn + GROUP () 1 1 1 + IDENT 1 u32 + PUNCT 1 - [joint] + PUNCT 1 > [alone] + PUNCT 1 & [joint] + PUNCT 1 ' [joint] + IDENT 1 a + IDENT 1 r#u32 + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT struct 42:Root[0000, 0]@0..6#ROOT2024 - IDENT S 42:Root[0000, 0]@7..8#ROOT2024 - PUNCH ; [alone] 42:Root[0000, 0]@8..9#ROOT2024 - - - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024"#]], + IDENT 42:Root[0000, 0]@0..6#ROOT2024 struct + IDENT 42:Root[0000, 0]@7..8#ROOT2024 S + GROUP {} 42:Root[0000, 0]@9..10#ROOT2024 42:Root[0000, 0]@46..47#ROOT2024 42:Root[0000, 0]@9..47#ROOT2024 + IDENT 42:Root[0000, 0]@11..16#ROOT2024 field + PUNCT 42:Root[0000, 0]@16..17#ROOT2024 : [alone] + PUNCT 42:Root[0000, 0]@18..19#ROOT2024 & [joint] + PUNCT 42:Root[0000, 0]@22..23#ROOT2024 ' [joint] + IDENT 42:Root[0000, 0]@22..24#ROOT2024 r#lt + IDENT 42:Root[0000, 0]@25..27#ROOT2024 fn + GROUP () 42:Root[0000, 0]@27..28#ROOT2024 42:Root[0000, 0]@31..32#ROOT2024 42:Root[0000, 0]@27..32#ROOT2024 + IDENT 42:Root[0000, 0]@28..31#ROOT2024 u32 + PUNCT 42:Root[0000, 0]@33..34#ROOT2024 - [joint] + PUNCT 42:Root[0000, 0]@34..35#ROOT2024 > [alone] + PUNCT 42:Root[0000, 0]@36..37#ROOT2024 & [joint] + PUNCT 42:Root[0000, 0]@38..39#ROOT2024 ' [joint] + IDENT 42:Root[0000, 0]@38..39#ROOT2024 a + IDENT 42:Root[0000, 0]@42..45#ROOT2024 r#u32 + "#]], ); } @@ -36,35 +56,37 @@ fn test_derive_empty() { fn test_derive_error() { assert_expand( "DeriveError", - r#"struct S;"#, + r#"struct S { field: u32 }"#, expect![[r#" - SUBTREE $$ 1 1 - IDENT struct 1 - IDENT S 1 - PUNCH ; [alone] 1 - - - - SUBTREE $$ 1 1 - IDENT compile_error 1 - PUNCH ! [alone] 1 - SUBTREE () 1 1 - LITERAL Str #[derive(DeriveError)] struct S ; 1 - PUNCH ; [alone] 1"#]], + IDENT 1 struct + IDENT 1 S + GROUP {} 1 1 1 + IDENT 1 field + PUNCT 1 : [alone] + IDENT 1 u32 + + + IDENT 1 compile_error + PUNCT 1 ! [joint] + GROUP () 1 1 1 + LITER 1 Str #[derive(DeriveError)] struct S {field 58 u32 } + PUNCT 1 ; [alone] + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT struct 42:Root[0000, 0]@0..6#ROOT2024 - IDENT S 42:Root[0000, 0]@7..8#ROOT2024 - PUNCH ; [alone] 42:Root[0000, 0]@8..9#ROOT2024 - - - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT compile_error 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH ! [alone] 42:Root[0000, 0]@0..100#ROOT2024 - SUBTREE () 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Str #[derive(DeriveError)] struct S ; 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH ; [alone] 42:Root[0000, 0]@0..100#ROOT2024"#]], + IDENT 42:Root[0000, 0]@0..6#ROOT2024 struct + IDENT 42:Root[0000, 0]@7..8#ROOT2024 S + GROUP {} 42:Root[0000, 0]@9..10#ROOT2024 42:Root[0000, 0]@22..23#ROOT2024 42:Root[0000, 0]@9..23#ROOT2024 + IDENT 42:Root[0000, 0]@11..16#ROOT2024 field + PUNCT 42:Root[0000, 0]@16..17#ROOT2024 : [alone] + IDENT 42:Root[0000, 0]@18..21#ROOT2024 u32 + + + IDENT 42:Root[0000, 0]@0..13#ROOT2024 compile_error + PUNCT 42:Root[0000, 0]@13..14#ROOT2024 ! [joint] + GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@64..65#ROOT2024 42:Root[0000, 0]@14..65#ROOT2024 + LITER 42:Root[0000, 0]@15..64#ROOT2024 Str #[derive(DeriveError)] struct S {field 58 u32 } + PUNCT 42:Root[0000, 0]@65..66#ROOT2024 ; [alone] + "#]], ); } @@ -74,45 +96,41 @@ fn test_fn_like_macro_noop() { "fn_like_noop", r#"ident, 0, 1, []"#, expect![[r#" - SUBTREE $$ 1 1 - IDENT ident 1 - PUNCH , [alone] 1 - LITERAL Integer 0 1 - PUNCH , [alone] 1 - LITERAL Integer 1 1 - PUNCH , [alone] 1 - SUBTREE [] 1 1 - - - - SUBTREE $$ 1 1 - IDENT ident 1 - PUNCH , [alone] 1 - LITERAL Integer 0 1 - PUNCH , [alone] 1 - LITERAL Integer 1 1 - PUNCH , [alone] 1 - SUBTREE [] 1 1"#]], + IDENT 1 ident + PUNCT 1 , [alone] + LITER 1 Integer 0 + PUNCT 1 , [alone] + LITER 1 Integer 1 + PUNCT 1 , [alone] + GROUP [] 1 1 1 + + + IDENT 1 ident + PUNCT 1 , [alone] + LITER 1 Integer 0 + PUNCT 1 , [alone] + LITER 1 Integer 1 + PUNCT 1 , [alone] + GROUP [] 1 1 1 + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT ident 42:Root[0000, 0]@0..5#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@5..6#ROOT2024 - LITERAL Integer 0 42:Root[0000, 0]@7..8#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@8..9#ROOT2024 - LITERAL Integer 1 42:Root[0000, 0]@10..11#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@11..12#ROOT2024 - SUBTREE [] 42:Root[0000, 0]@13..14#ROOT2024 42:Root[0000, 0]@14..15#ROOT2024 - - - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT ident 42:Root[0000, 0]@0..5#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@5..6#ROOT2024 - LITERAL Integer 0 42:Root[0000, 0]@7..8#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@8..9#ROOT2024 - LITERAL Integer 1 42:Root[0000, 0]@10..11#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@11..12#ROOT2024 - SUBTREE [] 42:Root[0000, 0]@13..14#ROOT2024 42:Root[0000, 0]@14..15#ROOT2024"#]], + IDENT 42:Root[0000, 0]@0..5#ROOT2024 ident + PUNCT 42:Root[0000, 0]@5..6#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@7..8#ROOT2024 Integer 0 + PUNCT 42:Root[0000, 0]@8..9#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@10..11#ROOT2024 Integer 1 + PUNCT 42:Root[0000, 0]@11..12#ROOT2024 , [alone] + GROUP [] 42:Root[0000, 0]@13..14#ROOT2024 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@13..15#ROOT2024 + + + IDENT 42:Root[0000, 0]@0..5#ROOT2024 ident + PUNCT 42:Root[0000, 0]@5..6#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@7..8#ROOT2024 Integer 0 + PUNCT 42:Root[0000, 0]@8..9#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@10..11#ROOT2024 Integer 1 + PUNCT 42:Root[0000, 0]@11..12#ROOT2024 , [alone] + GROUP [] 42:Root[0000, 0]@13..14#ROOT2024 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@13..15#ROOT2024 + "#]], ); } @@ -122,29 +140,25 @@ fn test_fn_like_macro_clone_ident_subtree() { "fn_like_clone_tokens", r#"ident, []"#, expect![[r#" - SUBTREE $$ 1 1 - IDENT ident 1 - PUNCH , [alone] 1 - SUBTREE [] 1 1 + IDENT 1 ident + PUNCT 1 , [alone] + GROUP [] 1 1 1 - - SUBTREE $$ 1 1 - IDENT ident 1 - PUNCH , [alone] 1 - SUBTREE [] 1 1"#]], + IDENT 1 ident + PUNCT 1 , [alone] + GROUP [] 1 1 1 + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT ident 42:Root[0000, 0]@0..5#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@5..6#ROOT2024 - SUBTREE [] 42:Root[0000, 0]@7..8#ROOT2024 42:Root[0000, 0]@8..9#ROOT2024 - + IDENT 42:Root[0000, 0]@0..5#ROOT2024 ident + PUNCT 42:Root[0000, 0]@5..6#ROOT2024 , [alone] + GROUP [] 42:Root[0000, 0]@7..8#ROOT2024 42:Root[0000, 0]@8..9#ROOT2024 42:Root[0000, 0]@7..9#ROOT2024 - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT ident 42:Root[0000, 0]@0..5#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@5..6#ROOT2024 - SUBTREE [] 42:Root[0000, 0]@7..9#ROOT2024 42:Root[0000, 0]@7..9#ROOT2024"#]], + IDENT 42:Root[0000, 0]@0..5#ROOT2024 ident + PUNCT 42:Root[0000, 0]@5..6#ROOT2024 , [alone] + GROUP [] 42:Root[0000, 0]@7..9#ROOT2024 42:Root[0000, 0]@7..9#ROOT2024 42:Root[0000, 0]@7..9#ROOT2024 + "#]], ); } @@ -154,21 +168,17 @@ fn test_fn_like_macro_clone_raw_ident() { "fn_like_clone_tokens", "r#async", expect![[r#" - SUBTREE $$ 1 1 - IDENT r#async 1 - + IDENT 1 r#async - SUBTREE $$ 1 1 - IDENT r#async 1"#]], + IDENT 1 r#async + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT r#async 42:Root[0000, 0]@0..7#ROOT2024 + IDENT 42:Root[0000, 0]@2..7#ROOT2024 r#async - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT r#async 42:Root[0000, 0]@0..7#ROOT2024"#]], + IDENT 42:Root[0000, 0]@2..7#ROOT2024 r#async + "#]], ); } @@ -178,23 +188,19 @@ fn test_fn_like_fn_like_span_join() { "fn_like_span_join", "foo bar", expect![[r#" - SUBTREE $$ 1 1 - IDENT foo 1 - IDENT bar 1 - + IDENT 1 foo + IDENT 1 bar - SUBTREE $$ 1 1 - IDENT r#joined 1"#]], + IDENT 1 r#joined + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT foo 42:Root[0000, 0]@0..3#ROOT2024 - IDENT bar 42:Root[0000, 0]@8..11#ROOT2024 - + IDENT 42:Root[0000, 0]@0..3#ROOT2024 foo + IDENT 42:Root[0000, 0]@8..11#ROOT2024 bar - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT r#joined 42:Root[0000, 0]@0..11#ROOT2024"#]], + IDENT 42:Root[0000, 0]@0..11#ROOT2024 r#joined + "#]], ); } @@ -204,29 +210,25 @@ fn test_fn_like_fn_like_span_ops() { "fn_like_span_ops", "set_def_site resolved_at_def_site start_span", expect![[r#" - SUBTREE $$ 1 1 - IDENT set_def_site 1 - IDENT resolved_at_def_site 1 - IDENT start_span 1 + IDENT 1 set_def_site + IDENT 1 resolved_at_def_site + IDENT 1 start_span - - SUBTREE $$ 1 1 - IDENT set_def_site 0 - IDENT resolved_at_def_site 1 - IDENT start_span 1"#]], + IDENT 0 set_def_site + IDENT 1 resolved_at_def_site + IDENT 1 start_span + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT set_def_site 42:Root[0000, 0]@0..12#ROOT2024 - IDENT resolved_at_def_site 42:Root[0000, 0]@13..33#ROOT2024 - IDENT start_span 42:Root[0000, 0]@34..44#ROOT2024 - + IDENT 42:Root[0000, 0]@0..12#ROOT2024 set_def_site + IDENT 42:Root[0000, 0]@13..33#ROOT2024 resolved_at_def_site + IDENT 42:Root[0000, 0]@34..44#ROOT2024 start_span - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT set_def_site 41:Root[0000, 0]@0..150#ROOT2024 - IDENT resolved_at_def_site 42:Root[0000, 0]@13..33#ROOT2024 - IDENT start_span 42:Root[0000, 0]@34..34#ROOT2024"#]], + IDENT 41:Root[0000, 0]@0..150#ROOT2024 set_def_site + IDENT 42:Root[0000, 0]@13..33#ROOT2024 resolved_at_def_site + IDENT 42:Root[0000, 0]@34..34#ROOT2024 start_span + "#]], ); } @@ -236,51 +238,39 @@ fn test_fn_like_mk_literals() { "fn_like_mk_literals", r#""#, expect![[r#" - SUBTREE $$ 1 1 - - - - SUBTREE $$ 1 1 - LITERAL ByteStr byte_string 1 - LITERAL Char c 1 - LITERAL Str string 1 - LITERAL Str -string 1 - LITERAL CStr cstring 1 - LITERAL Float 3.14f64 1 - PUNCH - [alone] 1 - LITERAL Float 3.14f64 1 - LITERAL Float 3.14 1 - PUNCH - [alone] 1 - LITERAL Float 3.14 1 - LITERAL Integer 123i64 1 - PUNCH - [alone] 1 - LITERAL Integer 123i64 1 - LITERAL Integer 123 1 - PUNCH - [alone] 1 - LITERAL Integer 123 1"#]], + + + LITER 1 ByteStr byte_string + LITER 1 Char c + LITER 1 Str string + LITER 1 Str -string + LITER 1 CStr cstring + LITER 1 Float 3.14f64 + LITER 1 Float -3.14f64 + LITER 1 Float 3.14 + LITER 1 Float -3.14 + LITER 1 Integer 123i64 + LITER 1 Integer -123i64 + LITER 1 Integer 123 + LITER 1 Integer -123 + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - - - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL ByteStr byte_string 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Char c 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Str string 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Str -string 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL CStr cstring 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Float 3.14f64 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Float 3.14f64 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Float 3.14 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Float 3.14 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Integer 123i64 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Integer 123i64 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Integer 123 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Integer 123 42:Root[0000, 0]@0..100#ROOT2024"#]], + + + LITER 42:Root[0000, 0]@0..100#ROOT2024 ByteStr byte_string + LITER 42:Root[0000, 0]@0..100#ROOT2024 Char c + LITER 42:Root[0000, 0]@0..100#ROOT2024 Str string + LITER 42:Root[0000, 0]@0..100#ROOT2024 Str -string + LITER 42:Root[0000, 0]@0..100#ROOT2024 CStr cstring + LITER 42:Root[0000, 0]@0..100#ROOT2024 Float 3.14f64 + LITER 42:Root[0000, 0]@0..100#ROOT2024 Float -3.14f64 + LITER 42:Root[0000, 0]@0..100#ROOT2024 Float 3.14 + LITER 42:Root[0000, 0]@0..100#ROOT2024 Float -3.14 + LITER 42:Root[0000, 0]@0..100#ROOT2024 Integer 123i64 + LITER 42:Root[0000, 0]@0..100#ROOT2024 Integer -123i64 + LITER 42:Root[0000, 0]@0..100#ROOT2024 Integer 123 + LITER 42:Root[0000, 0]@0..100#ROOT2024 Integer -123 + "#]], ); } @@ -290,21 +280,17 @@ fn test_fn_like_mk_idents() { "fn_like_mk_idents", r#""#, expect![[r#" - SUBTREE $$ 1 1 - - SUBTREE $$ 1 1 - IDENT standard 1 - IDENT r#raw 1"#]], + IDENT 1 standard + IDENT 1 r#raw + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT standard 42:Root[0000, 0]@0..100#ROOT2024 - IDENT r#raw 42:Root[0000, 0]@0..100#ROOT2024"#]], + IDENT 42:Root[0000, 0]@0..100#ROOT2024 standard + IDENT 42:Root[0000, 0]@0..100#ROOT2024 r#raw + "#]], ); } @@ -314,97 +300,93 @@ fn test_fn_like_macro_clone_literals() { "fn_like_clone_tokens", r###"1u16, 2_u32, -4i64, 3.14f32, "hello bridge", "suffixed"suffix, r##"raw"##, 'a', b'b', c"null""###, expect![[r#" - SUBTREE $$ 1 1 - LITERAL Integer 1u16 1 - PUNCH , [alone] 1 - LITERAL Integer 2_u32 1 - PUNCH , [alone] 1 - PUNCH - [alone] 1 - LITERAL Integer 4i64 1 - PUNCH , [alone] 1 - LITERAL Float 3.14f32 1 - PUNCH , [alone] 1 - LITERAL Str hello bridge 1 - PUNCH , [alone] 1 - LITERAL Err(()) "suffixed"suffix 1 - PUNCH , [alone] 1 - LITERAL StrRaw(2) raw 1 - PUNCH , [alone] 1 - LITERAL Char a 1 - PUNCH , [alone] 1 - LITERAL Byte b 1 - PUNCH , [alone] 1 - LITERAL CStr null 1 - - - - SUBTREE $$ 1 1 - LITERAL Integer 1u16 1 - PUNCH , [alone] 1 - LITERAL Integer 2_u32 1 - PUNCH , [alone] 1 - PUNCH - [alone] 1 - LITERAL Integer 4i64 1 - PUNCH , [alone] 1 - LITERAL Float 3.14f32 1 - PUNCH , [alone] 1 - LITERAL Str hello bridge 1 - PUNCH , [alone] 1 - LITERAL Str suffixedsuffix 1 - PUNCH , [alone] 1 - LITERAL StrRaw(2) raw 1 - PUNCH , [alone] 1 - LITERAL Char a 1 - PUNCH , [alone] 1 - LITERAL Byte b 1 - PUNCH , [alone] 1 - LITERAL CStr null 1"#]], + LITER 1 Integer 1u16 + PUNCT 1 , [alone] + LITER 1 Integer 2_u32 + PUNCT 1 , [alone] + PUNCT 1 - [alone] + LITER 1 Integer 4i64 + PUNCT 1 , [alone] + LITER 1 Float 3.14f32 + PUNCT 1 , [alone] + LITER 1 Str hello bridge + PUNCT 1 , [alone] + LITER 1 Str suffixedsuffix + PUNCT 1 , [alone] + LITER 1 StrRaw(2) raw + PUNCT 1 , [alone] + LITER 1 Char a + PUNCT 1 , [alone] + LITER 1 Byte b + PUNCT 1 , [alone] + LITER 1 CStr null + + + LITER 1 Integer 1u16 + PUNCT 1 , [alone] + LITER 1 Integer 2_u32 + PUNCT 1 , [alone] + PUNCT 1 - [alone] + LITER 1 Integer 4i64 + PUNCT 1 , [alone] + LITER 1 Float 3.14f32 + PUNCT 1 , [alone] + LITER 1 Str hello bridge + PUNCT 1 , [alone] + LITER 1 Str suffixedsuffix + PUNCT 1 , [alone] + LITER 1 StrRaw(2) raw + PUNCT 1 , [alone] + LITER 1 Char a + PUNCT 1 , [alone] + LITER 1 Byte b + PUNCT 1 , [alone] + LITER 1 CStr null + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Integer 1u16 42:Root[0000, 0]@0..4#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@4..5#ROOT2024 - LITERAL Integer 2_u32 42:Root[0000, 0]@6..11#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@11..12#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@13..14#ROOT2024 - LITERAL Integer 4i64 42:Root[0000, 0]@14..18#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@18..19#ROOT2024 - LITERAL Float 3.14f32 42:Root[0000, 0]@20..27#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@27..28#ROOT2024 - LITERAL Str hello bridge 42:Root[0000, 0]@29..43#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@43..44#ROOT2024 - LITERAL Err(()) "suffixed"suffix 42:Root[0000, 0]@45..61#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@61..62#ROOT2024 - LITERAL StrRaw(2) raw 42:Root[0000, 0]@63..73#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@73..74#ROOT2024 - LITERAL Char a 42:Root[0000, 0]@75..78#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@78..79#ROOT2024 - LITERAL Byte b 42:Root[0000, 0]@80..84#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@84..85#ROOT2024 - LITERAL CStr null 42:Root[0000, 0]@86..93#ROOT2024 - - - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Integer 1u16 42:Root[0000, 0]@0..4#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@4..5#ROOT2024 - LITERAL Integer 2_u32 42:Root[0000, 0]@6..11#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@11..12#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@13..14#ROOT2024 - LITERAL Integer 4i64 42:Root[0000, 0]@14..18#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@18..19#ROOT2024 - LITERAL Float 3.14f32 42:Root[0000, 0]@20..27#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@27..28#ROOT2024 - LITERAL Str hello bridge 42:Root[0000, 0]@29..43#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@43..44#ROOT2024 - LITERAL Str suffixedsuffix 42:Root[0000, 0]@45..61#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@61..62#ROOT2024 - LITERAL StrRaw(2) raw 42:Root[0000, 0]@63..73#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@73..74#ROOT2024 - LITERAL Char a 42:Root[0000, 0]@75..78#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@78..79#ROOT2024 - LITERAL Byte b 42:Root[0000, 0]@80..84#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@84..85#ROOT2024 - LITERAL CStr null 42:Root[0000, 0]@86..93#ROOT2024"#]], + LITER 42:Root[0000, 0]@0..4#ROOT2024 Integer 1u16 + PUNCT 42:Root[0000, 0]@4..5#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@6..11#ROOT2024 Integer 2_u32 + PUNCT 42:Root[0000, 0]@11..12#ROOT2024 , [alone] + PUNCT 42:Root[0000, 0]@13..14#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@14..18#ROOT2024 Integer 4i64 + PUNCT 42:Root[0000, 0]@18..19#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@20..27#ROOT2024 Float 3.14f32 + PUNCT 42:Root[0000, 0]@27..28#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@29..43#ROOT2024 Str hello bridge + PUNCT 42:Root[0000, 0]@43..44#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@45..61#ROOT2024 Str suffixedsuffix + PUNCT 42:Root[0000, 0]@61..62#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@63..73#ROOT2024 StrRaw(2) raw + PUNCT 42:Root[0000, 0]@73..74#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@75..78#ROOT2024 Char a + PUNCT 42:Root[0000, 0]@78..79#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@80..84#ROOT2024 Byte b + PUNCT 42:Root[0000, 0]@84..85#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@86..93#ROOT2024 CStr null + + + LITER 42:Root[0000, 0]@0..4#ROOT2024 Integer 1u16 + PUNCT 42:Root[0000, 0]@4..5#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@6..11#ROOT2024 Integer 2_u32 + PUNCT 42:Root[0000, 0]@11..12#ROOT2024 , [alone] + PUNCT 42:Root[0000, 0]@13..14#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@14..18#ROOT2024 Integer 4i64 + PUNCT 42:Root[0000, 0]@18..19#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@20..27#ROOT2024 Float 3.14f32 + PUNCT 42:Root[0000, 0]@27..28#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@29..43#ROOT2024 Str hello bridge + PUNCT 42:Root[0000, 0]@43..44#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@45..61#ROOT2024 Str suffixedsuffix + PUNCT 42:Root[0000, 0]@61..62#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@63..73#ROOT2024 StrRaw(2) raw + PUNCT 42:Root[0000, 0]@73..74#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@75..78#ROOT2024 Char a + PUNCT 42:Root[0000, 0]@78..79#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@80..84#ROOT2024 Byte b + PUNCT 42:Root[0000, 0]@84..85#ROOT2024 , [alone] + LITER 42:Root[0000, 0]@86..93#ROOT2024 CStr null + "#]], ); } @@ -414,61 +396,57 @@ fn test_fn_like_macro_negative_literals() { "fn_like_clone_tokens", r###"-1u16, - 2_u32, -3.14f32, - 2.7"###, expect![[r#" - SUBTREE $$ 1 1 - PUNCH - [alone] 1 - LITERAL Integer 1u16 1 - PUNCH , [alone] 1 - PUNCH - [alone] 1 - LITERAL Integer 2_u32 1 - PUNCH , [alone] 1 - PUNCH - [alone] 1 - LITERAL Float 3.14f32 1 - PUNCH , [alone] 1 - PUNCH - [alone] 1 - LITERAL Float 2.7 1 - - - - SUBTREE $$ 1 1 - PUNCH - [alone] 1 - LITERAL Integer 1u16 1 - PUNCH , [alone] 1 - PUNCH - [alone] 1 - LITERAL Integer 2_u32 1 - PUNCH , [alone] 1 - PUNCH - [alone] 1 - LITERAL Float 3.14f32 1 - PUNCH , [alone] 1 - PUNCH - [alone] 1 - LITERAL Float 2.7 1"#]], + PUNCT 1 - [alone] + LITER 1 Integer 1u16 + PUNCT 1 , [alone] + PUNCT 1 - [alone] + LITER 1 Integer 2_u32 + PUNCT 1 , [alone] + PUNCT 1 - [alone] + LITER 1 Float 3.14f32 + PUNCT 1 , [alone] + PUNCT 1 - [alone] + LITER 1 Float 2.7 + + + PUNCT 1 - [alone] + LITER 1 Integer 1u16 + PUNCT 1 , [alone] + PUNCT 1 - [alone] + LITER 1 Integer 2_u32 + PUNCT 1 , [alone] + PUNCT 1 - [alone] + LITER 1 Float 3.14f32 + PUNCT 1 , [alone] + PUNCT 1 - [alone] + LITER 1 Float 2.7 + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@0..1#ROOT2024 - LITERAL Integer 1u16 42:Root[0000, 0]@1..5#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@5..6#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@7..8#ROOT2024 - LITERAL Integer 2_u32 42:Root[0000, 0]@9..14#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@14..15#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@16..17#ROOT2024 - LITERAL Float 3.14f32 42:Root[0000, 0]@17..24#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@24..25#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@26..27#ROOT2024 - LITERAL Float 2.7 42:Root[0000, 0]@28..31#ROOT2024 - - - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@0..1#ROOT2024 - LITERAL Integer 1u16 42:Root[0000, 0]@1..5#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@5..6#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@7..8#ROOT2024 - LITERAL Integer 2_u32 42:Root[0000, 0]@9..14#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@14..15#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@16..17#ROOT2024 - LITERAL Float 3.14f32 42:Root[0000, 0]@17..24#ROOT2024 - PUNCH , [alone] 42:Root[0000, 0]@24..25#ROOT2024 - PUNCH - [alone] 42:Root[0000, 0]@26..27#ROOT2024 - LITERAL Float 2.7 42:Root[0000, 0]@28..31#ROOT2024"#]], + PUNCT 42:Root[0000, 0]@0..1#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@1..5#ROOT2024 Integer 1u16 + PUNCT 42:Root[0000, 0]@5..6#ROOT2024 , [alone] + PUNCT 42:Root[0000, 0]@7..8#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@9..14#ROOT2024 Integer 2_u32 + PUNCT 42:Root[0000, 0]@14..15#ROOT2024 , [alone] + PUNCT 42:Root[0000, 0]@16..17#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@17..24#ROOT2024 Float 3.14f32 + PUNCT 42:Root[0000, 0]@24..25#ROOT2024 , [alone] + PUNCT 42:Root[0000, 0]@26..27#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@28..31#ROOT2024 Float 2.7 + + + PUNCT 42:Root[0000, 0]@0..1#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@1..5#ROOT2024 Integer 1u16 + PUNCT 42:Root[0000, 0]@5..6#ROOT2024 , [alone] + PUNCT 42:Root[0000, 0]@7..8#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@9..14#ROOT2024 Integer 2_u32 + PUNCT 42:Root[0000, 0]@14..15#ROOT2024 , [alone] + PUNCT 42:Root[0000, 0]@16..17#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@17..24#ROOT2024 Float 3.14f32 + PUNCT 42:Root[0000, 0]@24..25#ROOT2024 , [alone] + PUNCT 42:Root[0000, 0]@26..27#ROOT2024 - [alone] + LITER 42:Root[0000, 0]@28..31#ROOT2024 Float 2.7 + "#]], ); } @@ -482,40 +460,64 @@ fn test_attr_macro() { r#"mod m {}"#, r#"some arguments"#, expect![[r#" - SUBTREE $$ 1 1 - IDENT mod 1 - IDENT m 1 - SUBTREE {} 1 1 - - SUBTREE $$ 1 1 - IDENT some 1 - IDENT arguments 1 - - SUBTREE $$ 1 1 - IDENT compile_error 1 - PUNCH ! [alone] 1 - SUBTREE () 1 1 - LITERAL Str #[attr_error(some arguments)] mod m {} 1 - PUNCH ; [alone] 1"#]], + IDENT 1 mod + IDENT 1 m + GROUP {} 1 1 1 + + + IDENT 1 some + IDENT 1 arguments + + + IDENT 1 compile_error + PUNCT 1 ! [joint] + GROUP () 1 1 1 + LITER 1 Str #[attr_error(some arguments )] mod m {} + PUNCT 1 ; [alone] + "#]], expect![[r#" - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT mod 42:Root[0000, 0]@0..3#ROOT2024 - IDENT m 42:Root[0000, 0]@4..5#ROOT2024 - SUBTREE {} 42:Root[0000, 0]@6..7#ROOT2024 42:Root[0000, 0]@7..8#ROOT2024 - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT some 42:Root[0000, 0]@0..4#ROOT2024 - IDENT arguments 42:Root[0000, 0]@5..14#ROOT2024 - - SUBTREE $$ 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - IDENT compile_error 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH ! [alone] 42:Root[0000, 0]@0..100#ROOT2024 - SUBTREE () 42:Root[0000, 0]@0..100#ROOT2024 42:Root[0000, 0]@0..100#ROOT2024 - LITERAL Str #[attr_error(some arguments)] mod m {} 42:Root[0000, 0]@0..100#ROOT2024 - PUNCH ; [alone] 42:Root[0000, 0]@0..100#ROOT2024"#]], + IDENT 42:Root[0000, 0]@0..3#ROOT2024 mod + IDENT 42:Root[0000, 0]@4..5#ROOT2024 m + GROUP {} 42:Root[0000, 0]@6..7#ROOT2024 42:Root[0000, 0]@7..8#ROOT2024 42:Root[0000, 0]@6..8#ROOT2024 + + + IDENT 42:Root[0000, 0]@0..4#ROOT2024 some + IDENT 42:Root[0000, 0]@5..14#ROOT2024 arguments + + + IDENT 42:Root[0000, 0]@0..13#ROOT2024 compile_error + PUNCT 42:Root[0000, 0]@13..14#ROOT2024 ! [joint] + GROUP () 42:Root[0000, 0]@14..15#ROOT2024 42:Root[0000, 0]@56..57#ROOT2024 42:Root[0000, 0]@14..57#ROOT2024 + LITER 42:Root[0000, 0]@15..56#ROOT2024 Str #[attr_error(some arguments )] mod m {} + PUNCT 42:Root[0000, 0]@57..58#ROOT2024 ; [alone] + "#]], ); } +#[test] +#[should_panic = "called `Result::unwrap()` on an `Err` value: \"Mismatched token groups\""] +fn test_broken_input_unclosed_delim() { + assert_expand("fn_like_clone_tokens", r###"{"###, expect![[]], expect![[]]); +} + +#[test] +#[should_panic = "called `Result::unwrap()` on an `Err` value: \"Unexpected '}'\""] +fn test_broken_input_unopened_delim() { + assert_expand("fn_like_clone_tokens", r###"}"###, expect![[]], expect![[]]); +} + +#[test] +#[should_panic = "called `Result::unwrap()` on an `Err` value: \"Expected '}'\""] +fn test_broken_input_mismatched_delim() { + assert_expand("fn_like_clone_tokens", r###"(}"###, expect![[]], expect![[]]); +} + +#[test] +#[should_panic = "called `Result::unwrap()` on an `Err` value: \"Invalid identifier: `🪟`\""] +fn test_broken_input_unknowm_token() { + assert_expand("fn_like_clone_tokens", r###"🪟"###, expect![[]], expect![[]]); +} + /// Tests that we find and classify all proc macros correctly. #[test] fn list_test_macros() { diff --git a/crates/proc-macro-srv/src/tests/utils.rs b/crates/proc-macro-srv/src/tests/utils.rs index f5a76e30bbcb..1b12308ad6c5 100644 --- a/crates/proc-macro-srv/src/tests/utils.rs +++ b/crates/proc-macro-srv/src/tests/utils.rs @@ -1,31 +1,25 @@ //! utils used in proc-macro tests use expect_test::Expect; -use span::{EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext}; -use tt::TextRange; +use span::{ + EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TextRange, +}; -use crate::{EnvSnapshot, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path}; +use crate::{ + EnvSnapshot, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path, token_stream::TokenStream, +}; -fn parse_string(call_site: SpanId, src: &str) -> crate::server_impl::TokenStream { - crate::server_impl::TokenStream::with_subtree(crate::server_impl::TopSubtree( - syntax_bridge::parse_to_token_tree_static_span(span::Edition::CURRENT, call_site, src) - .unwrap() - .0 - .into_vec(), - )) +fn parse_string(call_site: SpanId, src: &str) -> TokenStream { + TokenStream::from_str(src, call_site).unwrap() } fn parse_string_spanned( anchor: SpanAnchor, call_site: SyntaxContext, src: &str, -) -> crate::server_impl::TokenStream { - crate::server_impl::TokenStream::with_subtree(crate::server_impl::TopSubtree( - syntax_bridge::parse_to_token_tree(span::Edition::CURRENT, anchor, call_site, src) - .unwrap() - .0 - .into_vec(), - )) +) -> TokenStream { + TokenStream::from_str(src, Span { range: TextRange::default(), anchor, ctx: call_site }) + .unwrap() } pub fn assert_expand( @@ -60,16 +54,18 @@ fn assert_expand_impl( let def_site = SpanId(0); let call_site = SpanId(1); let mixed_site = SpanId(2); - let input_ts = parse_string(call_site, input).into_subtree(call_site); - let attr_ts = attr.map(|attr| parse_string(call_site, attr).into_subtree(call_site)); + let input_ts = parse_string(call_site, input); + let attr_ts = attr.map(|attr| parse_string(call_site, attr)); let input_ts_string = format!("{input_ts:?}"); let attr_ts_string = attr_ts.as_ref().map(|it| format!("{it:?}")); let res = expander.expand(macro_name, input_ts, attr_ts, def_site, call_site, mixed_site).unwrap(); expect.assert_eq(&format!( - "{input_ts_string}\n\n{}\n\n{res:?}", - attr_ts_string.unwrap_or_default() + "{input_ts_string}{}{}{}", + if attr_ts_string.is_some() { "\n\n" } else { "" }, + attr_ts_string.unwrap_or_default(), + if res.is_empty() { String::new() } else { format!("\n\n{res:?}") } )); let def_site = Span { @@ -90,17 +86,18 @@ fn assert_expand_impl( }; let mixed_site = call_site; - let fixture = - parse_string_spanned(call_site.anchor, call_site.ctx, input).into_subtree(call_site); - let attr = attr.map(|attr| { - parse_string_spanned(call_site.anchor, call_site.ctx, attr).into_subtree(call_site) - }); + let fixture = parse_string_spanned(call_site.anchor, call_site.ctx, input); + let attr = attr.map(|attr| parse_string_spanned(call_site.anchor, call_site.ctx, attr)); let fixture_string = format!("{fixture:?}"); let attr_string = attr.as_ref().map(|it| format!("{it:?}")); let res = expander.expand(macro_name, fixture, attr, def_site, call_site, mixed_site).unwrap(); - expect_spanned - .assert_eq(&format!("{fixture_string}\n\n{}\n\n{res:#?}", attr_string.unwrap_or_default())); + expect_spanned.assert_eq(&format!( + "{fixture_string}{}{}{}", + if attr_string.is_some() { "\n\n" } else { "" }, + attr_string.unwrap_or_default(), + if res.is_empty() { String::new() } else { format!("\n\n{res:?}") } + )); } pub(crate) fn list() -> Vec { diff --git a/crates/proc-macro-srv/src/token_stream.rs b/crates/proc-macro-srv/src/token_stream.rs new file mode 100644 index 000000000000..628d6942392c --- /dev/null +++ b/crates/proc-macro-srv/src/token_stream.rs @@ -0,0 +1,745 @@ +//! The proc-macro server token stream implementation. + +use core::fmt; +use std::sync::Arc; + +use intern::Symbol; +use proc_macro::Delimiter; +use rustc_lexer::{DocStyle, LiteralKind}; + +use crate::bridge::{DelimSpan, Group, Ident, LitKind, Literal, Punct, TokenTree}; + +/// Trait for allowing tests to parse tokenstreams with dynamic span ranges +pub(crate) trait SpanLike { + fn derive_ranged(&self, range: std::ops::Range) -> Self; +} + +#[derive(Clone)] +pub struct TokenStream(pub(crate) Arc>>); + +impl Default for TokenStream { + fn default() -> Self { + Self(Default::default()) + } +} + +impl TokenStream { + pub fn new(tts: Vec>) -> TokenStream { + TokenStream(Arc::new(tts)) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn iter(&self) -> TokenStreamIter<'_, S> { + TokenStreamIter::new(self) + } + + pub(crate) fn from_str(s: &str, span: S) -> Result + where + S: SpanLike + Copy, + { + let mut groups = Vec::new(); + groups.push((proc_macro::Delimiter::None, 0..0, vec![])); + let mut offset = 0; + let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No).peekable(); + while let Some(token) = tokens.next() { + let range = offset..offset + token.len as usize; + offset += token.len as usize; + + let mut is_joint = || { + tokens.peek().is_some_and(|token| { + matches!( + token.kind, + rustc_lexer::TokenKind::RawLifetime + | rustc_lexer::TokenKind::GuardedStrPrefix + | rustc_lexer::TokenKind::Lifetime { .. } + | rustc_lexer::TokenKind::Semi + | rustc_lexer::TokenKind::Comma + | rustc_lexer::TokenKind::Dot + | rustc_lexer::TokenKind::OpenParen + | rustc_lexer::TokenKind::CloseParen + | rustc_lexer::TokenKind::OpenBrace + | rustc_lexer::TokenKind::CloseBrace + | rustc_lexer::TokenKind::OpenBracket + | rustc_lexer::TokenKind::CloseBracket + | rustc_lexer::TokenKind::At + | rustc_lexer::TokenKind::Pound + | rustc_lexer::TokenKind::Tilde + | rustc_lexer::TokenKind::Question + | rustc_lexer::TokenKind::Colon + | rustc_lexer::TokenKind::Dollar + | rustc_lexer::TokenKind::Eq + | rustc_lexer::TokenKind::Bang + | rustc_lexer::TokenKind::Lt + | rustc_lexer::TokenKind::Gt + | rustc_lexer::TokenKind::Minus + | rustc_lexer::TokenKind::And + | rustc_lexer::TokenKind::Or + | rustc_lexer::TokenKind::Plus + | rustc_lexer::TokenKind::Star + | rustc_lexer::TokenKind::Slash + | rustc_lexer::TokenKind::Percent + | rustc_lexer::TokenKind::Caret + ) + }) + }; + + let Some((open_delim, _, tokenstream)) = groups.last_mut() else { + return Err("Unbalanced delimiters".to_owned()); + }; + match token.kind { + rustc_lexer::TokenKind::OpenParen => { + groups.push((proc_macro::Delimiter::Parenthesis, range, vec![])) + } + rustc_lexer::TokenKind::CloseParen if *open_delim != Delimiter::Parenthesis => { + return if *open_delim == Delimiter::None { + Err("Unexpected ')'".to_owned()) + } else { + Err("Expected ')'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseParen => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter, + stream: if stream.is_empty() { + None + } else { + Some(TokenStream::new(stream)) + }, + span: DelimSpan { + entire: span.derive_ranged(open_range.start..range.end), + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + }, + }), + ); + } + rustc_lexer::TokenKind::OpenBrace => { + groups.push((proc_macro::Delimiter::Brace, range, vec![])) + } + rustc_lexer::TokenKind::CloseBrace if *open_delim != Delimiter::Brace => { + return if *open_delim == Delimiter::None { + Err("Unexpected '}'".to_owned()) + } else { + Err("Expected '}'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseBrace => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter, + stream: if stream.is_empty() { + None + } else { + Some(TokenStream::new(stream)) + }, + span: DelimSpan { + entire: span.derive_ranged(open_range.start..range.end), + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + }, + }), + ); + } + rustc_lexer::TokenKind::OpenBracket => { + groups.push((proc_macro::Delimiter::Bracket, range, vec![])) + } + rustc_lexer::TokenKind::CloseBracket if *open_delim != Delimiter::Bracket => { + return if *open_delim == Delimiter::None { + Err("Unexpected ']'".to_owned()) + } else { + Err("Expected ']'".to_owned()) + }; + } + rustc_lexer::TokenKind::CloseBracket => { + let (delimiter, open_range, stream) = groups.pop().unwrap(); + groups.last_mut().ok_or_else(|| "Unbalanced delimiters".to_owned())?.2.push( + TokenTree::Group(Group { + delimiter, + stream: if stream.is_empty() { + None + } else { + Some(TokenStream::new(stream)) + }, + span: DelimSpan { + entire: span.derive_ranged(open_range.start..range.end), + open: span.derive_ranged(open_range), + close: span.derive_ranged(range), + }, + }), + ); + } + rustc_lexer::TokenKind::LineComment { doc_style: None } + | rustc_lexer::TokenKind::BlockComment { doc_style: None, terminated: _ } => { + continue; + } + rustc_lexer::TokenKind::LineComment { doc_style: Some(doc_style) } => { + let text = &s[range.start + 2..range.end]; + tokenstream.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); + if doc_style == DocStyle::Inner { + tokenstream.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span })); + } + tokenstream.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream: Some(TokenStream::new(vec![ + TokenTree::Ident(Ident { + sym: Symbol::intern("doc"), + is_raw: false, + span, + }), + TokenTree::Punct(Punct { ch: b'=', joint: false, span }), + TokenTree::Literal(Literal { + kind: LitKind::Str, + symbol: Symbol::intern(&text.escape_debug().to_string()), + suffix: None, + span: span.derive_ranged(range), + }), + ])), + span: DelimSpan { open: span, close: span, entire: span }, + })); + } + rustc_lexer::TokenKind::BlockComment { doc_style: Some(doc_style), terminated } => { + let text = + &s[range.start + 2..if terminated { range.end - 2 } else { range.end }]; + tokenstream.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); + if doc_style == DocStyle::Inner { + tokenstream.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span })); + } + tokenstream.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream: Some(TokenStream::new(vec![ + TokenTree::Ident(Ident { + sym: Symbol::intern("doc"), + is_raw: false, + span, + }), + TokenTree::Punct(Punct { ch: b'=', joint: false, span }), + TokenTree::Literal(Literal { + kind: LitKind::Str, + symbol: Symbol::intern(&text.escape_debug().to_string()), + suffix: None, + span: span.derive_ranged(range), + }), + ])), + span: DelimSpan { open: span, close: span, entire: span }, + })); + } + rustc_lexer::TokenKind::Whitespace => continue, + rustc_lexer::TokenKind::Frontmatter { .. } => unreachable!(), + rustc_lexer::TokenKind::Unknown => { + return Err(format!("Unknown token: `{}`", &s[range])); + } + rustc_lexer::TokenKind::UnknownPrefix => { + return Err(format!("Unknown prefix: `{}`", &s[range])); + } + rustc_lexer::TokenKind::UnknownPrefixLifetime => { + return Err(format!("Unknown lifetime prefix: `{}`", &s[range])); + } + // FIXME: Error on edition >= 2024 ... I dont think the proc-macro server can fetch editions currently + // and whose edition is this? + rustc_lexer::TokenKind::GuardedStrPrefix => { + tokenstream.push(TokenTree::Punct(Punct { + ch: s.as_bytes()[range.start], + joint: true, + span: span.derive_ranged(range.start..range.start + 1), + })); + tokenstream.push(TokenTree::Punct(Punct { + ch: s.as_bytes()[range.start + 1], + joint: is_joint(), + span: span.derive_ranged(range.start + 1..range.end), + })) + } + rustc_lexer::TokenKind::Ident => tokenstream.push(TokenTree::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: false, + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::InvalidIdent => { + return Err(format!("Invalid identifier: `{}`", &s[range])); + } + rustc_lexer::TokenKind::RawIdent => { + let range = range.start + 2..range.end; + tokenstream.push(TokenTree::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: true, + span: span.derive_ranged(range), + })) + } + rustc_lexer::TokenKind::Literal { kind, suffix_start } => { + tokenstream.push(TokenTree::Literal(literal_from_lexer( + &s[range.clone()], + span.derive_ranged(range), + kind, + suffix_start, + ))) + } + rustc_lexer::TokenKind::RawLifetime => { + let range = range.start + 1 + 2..range.end; + tokenstream.push(TokenTree::Punct(Punct { + ch: b'\'', + joint: true, + span: span.derive_ranged(range.start..range.start + 1), + })); + tokenstream.push(TokenTree::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: true, + span: span.derive_ranged(range), + })) + } + rustc_lexer::TokenKind::Lifetime { starts_with_number } => { + if starts_with_number { + return Err("Lifetime cannot start with a number".to_owned()); + } + let range = range.start + 1..range.end; + tokenstream.push(TokenTree::Punct(Punct { + ch: b'\'', + joint: true, + span: span.derive_ranged(range.start..range.start + 1), + })); + tokenstream.push(TokenTree::Ident(Ident { + sym: Symbol::intern(&s[range.clone()]), + is_raw: false, + span: span.derive_ranged(range), + })) + } + rustc_lexer::TokenKind::Semi => tokenstream.push(TokenTree::Punct(Punct { + ch: b';', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Comma => tokenstream.push(TokenTree::Punct(Punct { + ch: b',', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Dot => tokenstream.push(TokenTree::Punct(Punct { + ch: b'.', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::At => tokenstream.push(TokenTree::Punct(Punct { + ch: b'@', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Pound => tokenstream.push(TokenTree::Punct(Punct { + ch: b'#', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Tilde => tokenstream.push(TokenTree::Punct(Punct { + ch: b'~', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Question => tokenstream.push(TokenTree::Punct(Punct { + ch: b'?', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Colon => tokenstream.push(TokenTree::Punct(Punct { + ch: b':', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Dollar => tokenstream.push(TokenTree::Punct(Punct { + ch: b'$', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Eq => tokenstream.push(TokenTree::Punct(Punct { + ch: b'=', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Bang => tokenstream.push(TokenTree::Punct(Punct { + ch: b'!', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Lt => tokenstream.push(TokenTree::Punct(Punct { + ch: b'<', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Gt => tokenstream.push(TokenTree::Punct(Punct { + ch: b'>', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Minus => tokenstream.push(TokenTree::Punct(Punct { + ch: b'-', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::And => tokenstream.push(TokenTree::Punct(Punct { + ch: b'&', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Or => tokenstream.push(TokenTree::Punct(Punct { + ch: b'|', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Plus => tokenstream.push(TokenTree::Punct(Punct { + ch: b'+', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Star => tokenstream.push(TokenTree::Punct(Punct { + ch: b'*', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Slash => tokenstream.push(TokenTree::Punct(Punct { + ch: b'/', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Caret => tokenstream.push(TokenTree::Punct(Punct { + ch: b'^', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Percent => tokenstream.push(TokenTree::Punct(Punct { + ch: b'%', + joint: is_joint(), + span: span.derive_ranged(range), + })), + rustc_lexer::TokenKind::Eof => break, + } + } + if let Some((Delimiter::None, _, tokentrees)) = groups.pop() + && groups.is_empty() + { + Ok(TokenStream::new(tokentrees)) + } else { + Err("Mismatched token groups".to_owned()) + } + } +} + +impl fmt::Display for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for tt in self.0.iter() { + display_token_tree(tt, f)?; + } + Ok(()) + } +} + +fn display_token_tree(tt: &TokenTree, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match tt { + TokenTree::Group(Group { delimiter, stream, span: _ }) => { + write!( + f, + "{}", + match delimiter { + proc_macro::Delimiter::Parenthesis => "(", + proc_macro::Delimiter::Brace => "{", + proc_macro::Delimiter::Bracket => "[", + proc_macro::Delimiter::None => "", + } + )?; + if let Some(stream) = stream { + write!(f, "{stream}")?; + } + write!( + f, + "{}", + match delimiter { + proc_macro::Delimiter::Parenthesis => ")", + proc_macro::Delimiter::Brace => "}", + proc_macro::Delimiter::Bracket => "]", + proc_macro::Delimiter::None => "", + } + )?; + } + TokenTree::Punct(Punct { ch, joint, span: _ }) => { + write!(f, "{ch}{}", if *joint { "" } else { " " })? + } + TokenTree::Ident(Ident { sym, is_raw, span: _ }) => { + if *is_raw { + write!(f, "r#")?; + } + write!(f, "{sym} ")?; + } + TokenTree::Literal(lit) => { + display_fmt_literal(lit, f)?; + let joint = match lit.kind { + LitKind::Str + | LitKind::StrRaw(_) + | LitKind::ByteStr + | LitKind::ByteStrRaw(_) + | LitKind::CStr + | LitKind::CStrRaw(_) => true, + _ => false, + }; + if !joint { + write!(f, " ")?; + } + } + } + Ok(()) +} + +pub fn literal_to_string(literal: &Literal) -> String { + let mut buf = String::new(); + display_fmt_literal(literal, &mut buf).unwrap(); + buf +} + +fn display_fmt_literal(literal: &Literal, f: &mut impl std::fmt::Write) -> fmt::Result { + match literal.kind { + LitKind::Byte => write!(f, "b'{}'", literal.symbol), + LitKind::Char => write!(f, "'{}'", literal.symbol), + LitKind::Integer | LitKind::Float | LitKind::ErrWithGuar => { + write!(f, "{}", literal.symbol) + } + LitKind::Str => write!(f, "\"{}\"", literal.symbol), + LitKind::ByteStr => write!(f, "b\"{}\"", literal.symbol), + LitKind::CStr => write!(f, "c\"{}\"", literal.symbol), + LitKind::StrRaw(num_of_hashes) => { + let num_of_hashes = num_of_hashes as usize; + write!( + f, + r#"r{0:# { + let num_of_hashes = num_of_hashes as usize; + write!( + f, + r#"br{0:# { + let num_of_hashes = num_of_hashes as usize; + write!( + f, + r#"cr{0:# fmt::Debug for TokenStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + debug_token_stream(self, 0, f) + } +} + +fn debug_token_stream( + ts: &TokenStream, + depth: usize, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + for tt in ts.0.iter() { + debug_token_tree(tt, depth, f)?; + } + Ok(()) +} + +fn debug_token_tree( + tt: &TokenTree, + depth: usize, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + write!(f, "{:indent$}", "", indent = depth * 2)?; + match tt { + TokenTree::Group(Group { delimiter, stream, span }) => { + writeln!( + f, + "GROUP {}{} {:#?} {:#?} {:#?}", + match delimiter { + proc_macro::Delimiter::Parenthesis => "(", + proc_macro::Delimiter::Brace => "{", + proc_macro::Delimiter::Bracket => "[", + proc_macro::Delimiter::None => "$", + }, + match delimiter { + proc_macro::Delimiter::Parenthesis => ")", + proc_macro::Delimiter::Brace => "}", + proc_macro::Delimiter::Bracket => "]", + proc_macro::Delimiter::None => "$", + }, + span.open, + span.close, + span.entire, + )?; + if let Some(stream) = stream { + debug_token_stream(stream, depth + 1, f)?; + } + return Ok(()); + } + TokenTree::Punct(Punct { ch, joint, span }) => write!( + f, + "PUNCT {span:#?} {} {}", + *ch as char, + if *joint { "[joint]" } else { "[alone]" } + )?, + TokenTree::Ident(Ident { sym, is_raw, span }) => { + write!(f, "IDENT {span:#?} ")?; + if *is_raw { + write!(f, "r#")?; + } + write!(f, "{sym}")?; + } + TokenTree::Literal(Literal { kind, symbol, suffix, span }) => write!( + f, + "LITER {span:#?} {kind:?} {symbol}{}", + match suffix { + Some(suffix) => suffix.clone(), + None => Symbol::intern(""), + } + )?, + } + writeln!(f) +} + +impl TokenStream { + /// Push `tt` onto the end of the stream, possibly gluing it to the last + /// token. Uses `make_mut` to maximize efficiency. + pub(crate) fn push_tree(&mut self, tt: TokenTree) { + let vec_mut = Arc::make_mut(&mut self.0); + vec_mut.push(tt); + } + + /// Push `stream` onto the end of the stream, possibly gluing the first + /// token tree to the last token. (No other token trees will be glued.) + /// Uses `make_mut` to maximize efficiency. + pub(crate) fn push_stream(&mut self, stream: TokenStream) { + let vec_mut = Arc::make_mut(&mut self.0); + + let stream_iter = stream.0.iter().cloned(); + + vec_mut.extend(stream_iter); + } +} + +impl FromIterator> for TokenStream { + fn from_iter>>(iter: I) -> Self { + TokenStream::new(iter.into_iter().collect::>>()) + } +} + +#[derive(Clone)] +pub struct TokenStreamIter<'t, S> { + stream: &'t TokenStream, + index: usize, +} + +impl<'t, S> TokenStreamIter<'t, S> { + fn new(stream: &'t TokenStream) -> Self { + TokenStreamIter { stream, index: 0 } + } +} + +impl<'t, S> Iterator for TokenStreamIter<'t, S> { + type Item = &'t TokenTree; + + fn next(&mut self) -> Option<&'t TokenTree> { + self.stream.0.get(self.index).map(|tree| { + self.index += 1; + tree + }) + } +} + +pub(super) fn literal_from_lexer( + s: &str, + span: Span, + kind: rustc_lexer::LiteralKind, + suffix_start: u32, +) -> Literal { + let (kind, start_offset, end_offset) = match kind { + LiteralKind::Int { .. } => (LitKind::Integer, 0, 0), + LiteralKind::Float { .. } => (LitKind::Float, 0, 0), + LiteralKind::Char { terminated } => (LitKind::Char, 1, terminated as usize), + LiteralKind::Byte { terminated } => (LitKind::Byte, 2, terminated as usize), + LiteralKind::Str { terminated } => (LitKind::Str, 1, terminated as usize), + LiteralKind::ByteStr { terminated } => (LitKind::ByteStr, 2, terminated as usize), + LiteralKind::CStr { terminated } => (LitKind::CStr, 2, terminated as usize), + LiteralKind::RawStr { n_hashes } => ( + LitKind::StrRaw(n_hashes.unwrap_or_default()), + 2 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + LiteralKind::RawByteStr { n_hashes } => ( + LitKind::ByteStrRaw(n_hashes.unwrap_or_default()), + 3 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + LiteralKind::RawCStr { n_hashes } => ( + LitKind::CStrRaw(n_hashes.unwrap_or_default()), + 3 + n_hashes.unwrap_or_default() as usize, + 1 + n_hashes.unwrap_or_default() as usize, + ), + }; + + let (lit, suffix) = s.split_at(suffix_start as usize); + let lit = &lit[start_offset..lit.len() - end_offset]; + let suffix = match suffix { + "" | "_" => None, + suffix => Some(Symbol::intern(suffix)), + }; + + Literal { kind, symbol: Symbol::intern(lit), suffix, span } +} + +impl SpanLike for crate::SpanId { + fn derive_ranged(&self, _: std::ops::Range) -> Self { + *self + } +} + +impl SpanLike for () { + fn derive_ranged(&self, _: std::ops::Range) -> Self { + *self + } +} + +impl SpanLike for crate::Span { + fn derive_ranged(&self, range: std::ops::Range) -> Self { + crate::Span { + range: span::TextRange::new( + span::TextSize::new(range.start as u32), + span::TextSize::new(range.end as u32), + ), + anchor: self.anchor, + ctx: self.ctx, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrip() { + let token_stream = TokenStream::from_str("struct T {\"string\"}", ()).unwrap(); + token_stream.to_string(); + assert_eq!(token_stream.to_string(), "struct T {\"string\"}"); + } +} diff --git a/crates/rust-analyzer/Cargo.toml b/crates/rust-analyzer/Cargo.toml index 782ec556142c..2e48c5a5a66c 100644 --- a/crates/rust-analyzer/Cargo.toml +++ b/crates/rust-analyzer/Cargo.toml @@ -48,7 +48,6 @@ tracing-subscriber.workspace = true tracing-tree.workspace = true triomphe.workspace = true toml.workspace = true -nohash-hasher.workspace = true walkdir = "2.5.0" semver.workspace = true memchr = "2.7.5" @@ -61,7 +60,6 @@ hir-def.workspace = true hir-ty.workspace = true hir.workspace = true ide-db.workspace = true -intern.workspace = true # This should only be used in CLI ide-ssr.workspace = true ide.workspace = true diff --git a/crates/rust-analyzer/build.rs b/crates/rust-analyzer/build.rs index 0fd381d61221..cc7f11259928 100644 --- a/crates/rust-analyzer/build.rs +++ b/crates/rust-analyzer/build.rs @@ -5,7 +5,6 @@ use std::{env, path::PathBuf, process::Command}; fn main() { set_rerun(); set_commit_info(); - println!("cargo::rustc-check-cfg=cfg(rust_analyzer)"); if option_env!("CFG_RELEASE").is_none() { println!("cargo:rustc-env=POKE_RA_DEVS=1"); } diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index f9a547f61138..ea0752250db1 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -826,58 +826,6 @@ impl fmt::Display for Ident { } } -impl Literal { - pub fn display_no_minus(&self) -> impl fmt::Display { - struct NoMinus<'a, S>(&'a Literal); - impl fmt::Display for NoMinus<'_, S> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let symbol = - self.0.symbol.as_str().strip_prefix('-').unwrap_or(self.0.symbol.as_str()); - match self.0.kind { - LitKind::Byte => write!(f, "b'{symbol}'"), - LitKind::Char => write!(f, "'{symbol}'"), - LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{symbol}"), - LitKind::Str => write!(f, "\"{symbol}\""), - LitKind::ByteStr => write!(f, "b\"{symbol}\""), - LitKind::CStr => write!(f, "c\"{symbol}\""), - LitKind::StrRaw(num_of_hashes) => { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"r{0:# { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"br{0:# { - let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"cr{0:# fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.kind { diff --git a/xtask/src/install.rs b/xtask/src/install.rs index 975e361ba50b..bddce0f027fc 100644 --- a/xtask/src/install.rs +++ b/xtask/src/install.rs @@ -174,10 +174,17 @@ fn install_server(sh: &Shell, opts: ServerOpt) -> anyhow::Result<()> { fn install_proc_macro_server(sh: &Shell, opts: ProcMacroServerOpt) -> anyhow::Result<()> { let profile = if opts.dev_rel { "dev-rel" } else { "release" }; - cmd!( + let mut cmd = cmd!( sh, - "cargo +nightly install --path crates/proc-macro-srv-cli --profile={profile} --locked --force --features sysroot-abi" - ).run()?; + "cargo install --path crates/proc-macro-srv-cli --profile={profile} --locked --force --features sysroot-abi" + ); + if std::env::var_os("RUSTUP_TOOLCHAIN").is_none() { + cmd = cmd.env("RUSTUP_TOOLCHAIN", "nightly"); + } else { + cmd = cmd.env("RUSTC_BOOTSTRAP", "1"); + } + + cmd.run()?; Ok(()) } diff --git a/xtask/src/tidy.rs b/xtask/src/tidy.rs index ebfc7d0a9430..05528505f280 100644 --- a/xtask/src/tidy.rs +++ b/xtask/src/tidy.rs @@ -194,6 +194,7 @@ fn check_test_attrs(path: &Path, text: &str) { "test-utils/src/fixture.rs", // Generated code from lints contains doc tests in string literals. "ide-db/src/generated/lints.rs", + "proc-macro-srv/src/tests/mod.rs", ]; if need_panic.iter().any(|p| path.ends_with(p)) { return;