itsscb/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-10-01 11:31:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Abtract away json protocol for proc-macro-srv

Browse Source
This commit is contained in:
Lukas Wirth 2025-07-30 15:31:18 +02:00
parent c81560c628
commit 4d5bb86ad7
14 changed files with 472 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Cargo.lock generated
View File

@ -23,6 +23,12 @@ version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "anstyle"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
[[package]]
name = "anyhow"
version = "1.0.98"
@ -287,6 +293,31 @@ dependencies = [
"tracing",
]
[[package]]
name = "clap"
version = "4.5.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.5.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966"
dependencies = [
"anstyle",
"clap_lex",
]
[[package]]
name = "clap_lex"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
[[package]]
name = "countme"
version = "3.0.1"
@ -1615,6 +1646,7 @@ dependencies = [
name = "proc-macro-srv-cli"
version = "0.0.0"
dependencies = [
"clap",
"proc-macro-api",
"proc-macro-srv",
"tt",

2
crates/load-cargo/src/lib.rs
View File

@ -533,7 +533,7 @@ impl ProcMacroExpander for Expander {
current_dir,
) {
Ok(Ok(subtree)) => Ok(subtree),
Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)),
Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err)),
Err(err) => Err(ProcMacroExpansionError::System(err.to_string())),
}
}

172
crates/proc-macro-api/src/legacy_protocol.rs Normal file
View File

@ -0,0 +1,172 @@
//! The initial proc-macro-srv protocol, soon to be deprecated.
pub mod json;
pub mod msg;
use std::{
io::{BufRead, Write},
sync::Arc,
};
use paths::AbsPath;
use span::Span;
use crate::{
ProcMacro, ProcMacroKind, ServerError,
legacy_protocol::{
json::{read_json, write_json},
msg::{
ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, Message, Request, Response,
ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map,
flat::serialize_span_data_index_map,
},
},
process::ProcMacroServerProcess,
version,
};
pub(crate) use crate::legacy_protocol::msg::SpanMode;
/// Legacy span type, only defined here as it is still used by the proc-macro server.
/// While rust-analyzer doesn't use this anymore at all, RustRover relies on the legacy type for
/// proc-macro expansion.
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct SpanId(pub u32);
impl std::fmt::Debug for SpanId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
pub(crate) fn version_check(srv: &ProcMacroServerProcess) -> Result<u32, ServerError> {
let request = Request::ApiVersionCheck {};
let response = send_task(srv, request)?;
match response {
Response::ApiVersionCheck(version) => Ok(version),
_ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
}
}
/// Enable support for rust-analyzer span mode if the server supports it.
pub(crate) fn enable_rust_analyzer_spans(
srv: &ProcMacroServerProcess,
) -> Result<SpanMode, ServerError> {
let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer });
let response = send_task(srv, request)?;
match response {
Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode),
_ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
}
}
/// Finds proc-macros in a given dynamic library.
pub(crate) fn find_proc_macros(
srv: &ProcMacroServerProcess,
dylib_path: &AbsPath,
) -> Result<Result<Vec<(String, ProcMacroKind)>, String>, ServerError> {
let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() };
let response = send_task(srv, request)?;
match response {
Response::ListMacros(it) => Ok(it),
_ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
}
}
pub(crate) fn expand(
proc_macro: &ProcMacro,
subtree: tt::SubtreeView<'_, Span>,
attr: Option<tt::SubtreeView<'_, Span>>,
env: Vec<(String, String)>,
def_site: Span,
call_site: Span,
mixed_site: Span,
current_dir: String,
) -> Result<Result<tt::TopSubtree<span::SpanData<span::SyntaxContext>>, String>, crate::ServerError>
{
let version = proc_macro.process.version();
let mut span_data_table = SpanDataIndexMap::default();
let def_site = span_data_table.insert_full(def_site).0;
let call_site = span_data_table.insert_full(call_site).0;
let mixed_site = span_data_table.insert_full(mixed_site).0;
let task = ExpandMacro {
data: ExpandMacroData {
macro_body: FlatTree::new(subtree, version, &mut span_data_table),
macro_name: proc_macro.name.to_string(),
attributes: attr.map(|subtree| FlatTree::new(subtree, version, &mut span_data_table)),
has_global_spans: ExpnGlobals {
serialize: version >= version::HAS_GLOBAL_SPANS,
def_site,
call_site,
mixed_site,
},
span_data_table: if proc_macro.process.rust_analyzer_spans() {
serialize_span_data_index_map(&span_data_table)
} else {
Vec::new()
},
},
lib: proc_macro.dylib_path.to_path_buf().into(),
env,
current_dir: Some(current_dir),
};
let response = send_task(&proc_macro.process, Request::ExpandMacro(Box::new(task)))?;
match response {
Response::ExpandMacro(it) => Ok(it
.map(|tree| {
let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table);
if proc_macro.needs_fixup_change() {
proc_macro.change_fixup_to_match_old_server(&mut expanded);
}
expanded
})
.map_err(|msg| msg.0)),
Response::ExpandMacroExtended(it) => Ok(it
.map(|resp| {
let mut expanded = FlatTree::to_subtree_resolved(
resp.tree,
version,
&deserialize_span_data_index_map(&resp.span_data_table),
);
if proc_macro.needs_fixup_change() {
proc_macro.change_fixup_to_match_old_server(&mut expanded);
}
expanded
})
.map_err(|msg| msg.0)),
_ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
}
}
/// Sends a request to the proc-macro server and waits for a response.
fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result<Response, ServerError> {
if let Some(server_error) = srv.exited() {
return Err(server_error.clone());
}
srv.send_task(send_request, req)
}
/// Sends a request to the server and reads the response.
fn send_request(
mut writer: &mut dyn Write,
mut reader: &mut dyn BufRead,
req: Request,
buf: &mut String,
) -> Result<Option<Response>, ServerError> {
req.write(write_json, &mut writer).map_err(|err| ServerError {
message: "failed to write request".into(),
io: Some(Arc::new(err)),
})?;
let res = Response::read(read_json, &mut reader, buf).map_err(|err| ServerError {
message: "failed to read response".into(),
io: Some(Arc::new(err)),
})?;
Ok(res)
}

25
crates/proc-macro-api/src/legacy_protocol/msg.rs
View File

@ -1,5 +1,6 @@
//! Defines messages for cross-process message passing based on `ndjson` wire protocol
pub(crate) mod flat;
pub use self::flat::*;
use std::io::{self, BufRead, Write};
@ -9,24 +10,6 @@ use serde_derive::{Deserialize, Serialize};
use crate::ProcMacroKind;
pub use self::flat::{
FlatTree, SpanDataIndexMap, deserialize_span_data_index_map, serialize_span_data_index_map,
};
pub use span::TokenId;
// The versions of the server protocol
pub const NO_VERSION_CHECK_VERSION: u32 = 0;
pub const VERSION_CHECK_VERSION: u32 = 1;
pub const ENCODE_CLOSE_SPAN_VERSION: u32 = 2;
pub const HAS_GLOBAL_SPANS: u32 = 3;
pub const RUST_ANALYZER_SPAN_SUPPORT: u32 = 4;
/// Whether literals encode their kind as an additional u32 field and idents their rawness as a u32 field.
pub const EXTENDED_LEAF_DATA: u32 = 5;
pub const HASHED_AST_ID: u32 = 6;
/// Current API version of the proc-macro protocol.
pub const CURRENT_API_VERSION: u32 = HASHED_AST_ID;
/// Represents requests sent from the client to the proc-macro-srv.
#[derive(Debug, Serialize, Deserialize)]
pub enum Request {
@ -48,7 +31,7 @@ pub enum Request {
}
/// Defines the mode used for handling span data.
#[derive(Copy, Clone, Default, Debug, Serialize, Deserialize)]
#[derive(Copy, Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub enum SpanMode {
/// Default mode, where spans are identified by an ID.
#[default]
@ -210,6 +193,8 @@ mod tests {
TopSubtreeBuilder,
};
use crate::version;
use super::*;
fn fixture_token_tree() -> TopSubtree<Span> {
@ -308,7 +293,7 @@ mod tests {
#[test]
fn test_proc_macro_rpc_works() {
let tt = fixture_token_tree();
for v in RUST_ANALYZER_SPAN_SUPPORT..=CURRENT_API_VERSION {
for v in version::RUST_ANALYZER_SPAN_SUPPORT..=version::CURRENT_API_VERSION {
let mut span_data_table = Default::default();
let task = ExpandMacro {
data: ExpandMacroData {

105
crates/proc-macro-api/src/legacy_protocol/msg/flat.rs
View File

@ -40,9 +40,12 @@ use std::collections::VecDeque;
use intern::Symbol;
use rustc_hash::FxHashMap;
use serde_derive::{Deserialize, Serialize};
use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange, TokenId};
use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange};
use crate::legacy_protocol::msg::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA};
use crate::{
legacy_protocol::SpanId,
version::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA},
};
pub type SpanDataIndexMap =
indexmap::IndexSet<Span, std::hash::BuildHasherDefault<rustc_hash::FxHasher>>;
@ -62,7 +65,7 @@ pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec<u32> {
}
pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap {
debug_assert!(map.len() % 5 == 0);
debug_assert!(map.len().is_multiple_of(5));
map.chunks_exact(5)
.map(|span| {
let &[file_id, ast_id, start, end, e] = span else { unreachable!() };
@ -91,27 +94,27 @@ pub struct FlatTree {
}
struct SubtreeRepr {
open: TokenId,
close: TokenId,
open: SpanId,
close: SpanId,
kind: tt::DelimiterKind,
tt: [u32; 2],
}
struct LiteralRepr {
id: TokenId,
id: SpanId,
text: u32,
suffix: u32,
kind: u16,
}
struct PunctRepr {
id: TokenId,
id: SpanId,
char: char,
spacing: tt::Spacing,
}
struct IdentRepr {
id: TokenId,
id: SpanId,
text: u32,
is_raw: bool,
}
@ -122,7 +125,7 @@ impl FlatTree {
version: u32,
span_data_table: &mut SpanDataIndexMap,
) -> FlatTree {
let mut w = Writer {
let mut w = Writer::<Span> {
string_table: FxHashMap::default(),
work: VecDeque::new(),
span_data_table,
@ -159,8 +162,11 @@ impl FlatTree {
}
}
pub fn new_raw(subtree: tt::SubtreeView<'_, TokenId>, version: u32) -> FlatTree {
let mut w = Writer {
pub fn new_raw<T: SpanTransformer<Table = ()>>(
subtree: tt::SubtreeView<'_, T::Span>,
version: u32,
) -> FlatTree {
let mut w = Writer::<T> {
string_table: FxHashMap::default(),
work: VecDeque::new(),
span_data_table: &mut (),
@ -202,7 +208,7 @@ impl FlatTree {
version: u32,
span_data_table: &SpanDataIndexMap,
) -> tt::TopSubtree<Span> {
Reader {
Reader::<Span> {
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
read_vec(self.subtree, SubtreeRepr::read_with_close_span)
} else {
@ -227,8 +233,11 @@ impl FlatTree {
.read()
}
pub fn to_subtree_unresolved(self, version: u32) -> tt::TopSubtree<TokenId> {
Reader {
pub fn to_subtree_unresolved<T: SpanTransformer<Table = ()>>(
self,
version: u32,
) -> tt::TopSubtree<T::Span> {
Reader::<T> {
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
read_vec(self.subtree, SubtreeRepr::read_with_close_span)
} else {
@ -283,7 +292,7 @@ impl SubtreeRepr {
3 => tt::DelimiterKind::Bracket,
other => panic!("bad kind {other}"),
};
SubtreeRepr { open: TokenId(open), close: TokenId(!0), kind, tt: [lo, len] }
SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] }
}
fn write_with_close_span(self) -> [u32; 5] {
let kind = match self.kind {
@ -302,7 +311,7 @@ impl SubtreeRepr {
3 => tt::DelimiterKind::Bracket,
other => panic!("bad kind {other}"),
};
SubtreeRepr { open: TokenId(open), close: TokenId(close), kind, tt: [lo, len] }
SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] }
}
}
@ -311,13 +320,13 @@ impl LiteralRepr {
[self.id.0, self.text]
}
fn read([id, text]: [u32; 2]) -> LiteralRepr {
LiteralRepr { id: TokenId(id), text, kind: 0, suffix: !0 }
LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 }
}
fn write_with_kind(self) -> [u32; 4] {
[self.id.0, self.text, self.kind as u32, self.suffix]
}
fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr {
LiteralRepr { id: TokenId(id), text, kind: kind as u16, suffix }
LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix }
}
}
@ -335,7 +344,7 @@ impl PunctRepr {
1 => tt::Spacing::Joint,
other => panic!("bad spacing {other}"),
};
PunctRepr { id: TokenId(id), char: char.try_into().unwrap(), spacing }
PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing }
}
}
@ -344,44 +353,46 @@ impl IdentRepr {
[self.id.0, self.text]
}
fn read(data: [u32; 2]) -> IdentRepr {
IdentRepr { id: TokenId(data[0]), text: data[1], is_raw: false }
IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false }
}
fn write_with_rawness(self) -> [u32; 3] {
[self.id.0, self.text, self.is_raw as u32]
}
fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr {
IdentRepr { id: TokenId(id), text, is_raw: is_raw == 1 }
IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 }
}
}
trait InternableSpan: Copy {
pub trait SpanTransformer {
type Table;
fn token_id_of(table: &mut Self::Table, s: Self) -> TokenId;
fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self;
type Span: Copy;
fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId;
fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span;
}
impl InternableSpan for TokenId {
impl SpanTransformer for SpanId {
type Table = ();
fn token_id_of((): &mut Self::Table, token_id: Self) -> TokenId {
type Span = Self;
fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId {
token_id
}
fn span_for_token_id((): &Self::Table, id: TokenId) -> Self {
fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span {
id
}
}
impl InternableSpan for Span {
impl SpanTransformer for Span {
type Table = SpanDataIndexMap;
fn token_id_of(table: &mut Self::Table, span: Self) -> TokenId {
TokenId(table.insert_full(span).0 as u32)
type Span = Self;
fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId {
SpanId(table.insert_full(span).0 as u32)
}
fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self {
fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span {
*table.get_index(id.0 as usize).unwrap_or_else(|| &table[0])
}
}
struct Writer<'a, 'span, S: InternableSpan> {
work: VecDeque<(usize, tt::iter::TtIter<'a, S>)>,
struct Writer<'a, 'span, S: SpanTransformer> {
work: VecDeque<(usize, tt::iter::TtIter<'a, S::Span>)>,
string_table: FxHashMap<std::borrow::Cow<'a, str>, u32>,
span_data_table: &'span mut S::Table,
version: u32,
@ -394,8 +405,8 @@ struct Writer<'a, 'span, S: InternableSpan> {
text: Vec<String>,
}
impl<'a, S: InternableSpan> Writer<'a, '_, S> {
fn write(&mut self, root: tt::SubtreeView<'a, S>) {
impl<'a, T: SpanTransformer> Writer<'a, '_, T> {
fn write(&mut self, root: tt::SubtreeView<'a, T::Span>) {
let subtree = root.top_subtree();
self.enqueue(subtree, root.iter());
while let Some((idx, subtree)) = self.work.pop_front() {
@ -403,11 +414,11 @@ impl<'a, S: InternableSpan> Writer<'a, '_, S> {
}
}
fn token_id_of(&mut self, span: S) -> TokenId {
S::token_id_of(self.span_data_table, span)
fn token_id_of(&mut self, span: T::Span) -> SpanId {
T::token_id_of(self.span_data_table, span)
}
fn subtree(&mut self, idx: usize, subtree: tt::iter::TtIter<'a, S>) {
fn subtree(&mut self, idx: usize, subtree: tt::iter::TtIter<'a, T::Span>) {
let mut first_tt = self.token_tree.len();
let n_tt = subtree.clone().count(); // FIXME: `count()` walks over the entire iterator.
self.token_tree.resize(first_tt + n_tt, !0);
@ -478,7 +489,11 @@ impl<'a, S: InternableSpan> Writer<'a, '_, S> {
}
}
fn enqueue(&mut self, subtree: &'a tt::Subtree<S>, contents: tt::iter::TtIter<'a, S>) -> u32 {
fn enqueue(
&mut self,
subtree: &'a tt::Subtree<T::Span>,
contents: tt::iter::TtIter<'a, T::Span>,
) -> u32 {
let idx = self.subtree.len();
let open = self.token_id_of(subtree.delimiter.open);
let close = self.token_id_of(subtree.delimiter.close);
@ -507,7 +522,7 @@ impl<'a, S: InternableSpan> Writer<'a, '_, S> {
}
}
struct Reader<'span, S: InternableSpan> {
struct Reader<'span, S: SpanTransformer> {
version: u32,
subtree: Vec<SubtreeRepr>,
literal: Vec<LiteralRepr>,
@ -518,11 +533,11 @@ struct Reader<'span, S: InternableSpan> {
span_data_table: &'span S::Table,
}
impl<S: InternableSpan> Reader<'_, S> {
pub(crate) fn read(self) -> tt::TopSubtree<S> {
let mut res: Vec<Option<(tt::Delimiter<S>, Vec<tt::TokenTree<S>>)>> =
impl<T: SpanTransformer> Reader<'_, T> {
pub(crate) fn read(self) -> tt::TopSubtree<T::Span> {
let mut res: Vec<Option<(tt::Delimiter<T::Span>, Vec<tt::TokenTree<T::Span>>)>> =
vec![None; self.subtree.len()];
let read_span = |id| S::span_for_token_id(self.span_data_table, id);
let read_span = |id| T::span_for_token_id(self.span_data_table, id);
for i in (0..self.subtree.len()).rev() {
let repr = &self.subtree[i];
let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];

93
crates/proc-macro-api/src/lib.rs
View File

@ -5,24 +5,29 @@
//! is used to provide basic infrastructure for communication between two
//! processes: Client (RA itself), Server (the external program)
pub mod legacy_protocol {
pub mod json;
pub mod msg;
}
pub mod legacy_protocol;
mod process;
use paths::{AbsPath, AbsPathBuf};
use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span};
use std::{fmt, io, sync::Arc, time::SystemTime};
use crate::{
legacy_protocol::msg::{
ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, HAS_GLOBAL_SPANS, HASHED_AST_ID,
PanicMessage, RUST_ANALYZER_SPAN_SUPPORT, Request, Response, SpanDataIndexMap,
deserialize_span_data_index_map, flat::serialize_span_data_index_map,
},
process::ProcMacroServerProcess,
};
use crate::process::ProcMacroServerProcess;
/// The versions of the server protocol
pub mod version {
pub const NO_VERSION_CHECK_VERSION: u32 = 0;
pub const VERSION_CHECK_VERSION: u32 = 1;
pub const ENCODE_CLOSE_SPAN_VERSION: u32 = 2;
pub const HAS_GLOBAL_SPANS: u32 = 3;
pub const RUST_ANALYZER_SPAN_SUPPORT: u32 = 4;
/// Whether literals encode their kind as an additional u32 field and idents their rawness as a u32 field.
pub const EXTENDED_LEAF_DATA: u32 = 5;
pub const HASHED_AST_ID: u32 = 6;
/// Current API version of the proc-macro protocol.
pub const CURRENT_API_VERSION: u32 = HASHED_AST_ID;
}
/// Represents different kinds of procedural macros that can be expanded by the external server.
#[derive(Copy, Clone, Eq, PartialEq, Debug, serde_derive::Serialize, serde_derive::Deserialize)]
@ -163,7 +168,7 @@ impl ProcMacro {
fn needs_fixup_change(&self) -> bool {
let version = self.process.version();
(RUST_ANALYZER_SPAN_SUPPORT..HASHED_AST_ID).contains(&version)
(version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version)
}
/// On some server versions, the fixup ast id is different than ours. So change it to match.
@ -204,7 +209,7 @@ impl ProcMacro {
call_site: Span,
mixed_site: Span,
current_dir: String,
) -> Result<Result<tt::TopSubtree<Span>, PanicMessage>, ServerError> {
) -> Result<Result<tt::TopSubtree<Span>, String>, ServerError> {
let (mut subtree, mut attr) = (subtree, attr);
let (mut subtree_changed, mut attr_changed);
if self.needs_fixup_change() {
@ -219,57 +224,15 @@ impl ProcMacro {
}
}
let version = self.process.version();
let mut span_data_table = SpanDataIndexMap::default();
let def_site = span_data_table.insert_full(def_site).0;
let call_site = span_data_table.insert_full(call_site).0;
let mixed_site = span_data_table.insert_full(mixed_site).0;
let task = ExpandMacro {
data: ExpandMacroData {
macro_body: FlatTree::new(subtree, version, &mut span_data_table),
macro_name: self.name.to_string(),
attributes: attr
.map(|subtree| FlatTree::new(subtree, version, &mut span_data_table)),
has_global_spans: ExpnGlobals {
serialize: version >= HAS_GLOBAL_SPANS,
def_site,
call_site,
mixed_site,
},
span_data_table: if version >= RUST_ANALYZER_SPAN_SUPPORT {
serialize_span_data_index_map(&span_data_table)
} else {
Vec::new()
},
},
lib: self.dylib_path.to_path_buf().into(),
legacy_protocol::expand(
self,
subtree,
attr,
env,
current_dir: Some(current_dir),
};
let response = self.process.send_task(Request::ExpandMacro(Box::new(task)))?;
match response {
Response::ExpandMacro(it) => Ok(it.map(|tree| {
let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table);
if self.needs_fixup_change() {
self.change_fixup_to_match_old_server(&mut expanded);
}
expanded
})),
Response::ExpandMacroExtended(it) => Ok(it.map(|resp| {
let mut expanded = FlatTree::to_subtree_resolved(
resp.tree,
version,
&deserialize_span_data_index_map(&resp.span_data_table),
);
if self.needs_fixup_change() {
self.change_fixup_to_match_old_server(&mut expanded);
}
expanded
})),
_ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
}
def_site,
call_site,
mixed_site,
current_dir,
)
}
}

113
crates/proc-macro-api/src/process.rs
View File

@ -12,13 +12,8 @@ use stdx::JodChild;
use crate::{
ProcMacroKind, ServerError,
legacy_protocol::{
json::{read_json, write_json},
msg::{
CURRENT_API_VERSION, Message, RUST_ANALYZER_SPAN_SUPPORT, Request, Response,
ServerConfig, SpanMode,
},
},
legacy_protocol::{self, SpanMode},
version,
};
/// Represents a process handling proc-macro communication.
@ -28,11 +23,16 @@ pub(crate) struct ProcMacroServerProcess {
/// hence the lock on the state.
state: Mutex<ProcessSrvState>,
version: u32,
mode: SpanMode,
protocol: Protocol,
/// Populated when the server exits.
exited: OnceLock<AssertUnwindSafe<ServerError>>,
}
#[derive(Debug)]
enum Protocol {
LegacyJson { mode: SpanMode },
}
/// Maintains the state of the proc-macro server process.
#[derive(Debug)]
struct ProcessSrvState {
@ -56,27 +56,26 @@ impl ProcMacroServerProcess {
io::Result::Ok(ProcMacroServerProcess {
state: Mutex::new(ProcessSrvState { process, stdin, stdout }),
version: 0,
mode: SpanMode::Id,
protocol: Protocol::LegacyJson { mode: SpanMode::Id },
exited: OnceLock::new(),
})
};
let mut srv = create_srv()?;
tracing::info!("sending proc-macro server version check");
match srv.version_check() {
Ok(v) if v > CURRENT_API_VERSION => Err(io::Error::other(
format!( "The version of the proc-macro server ({v}) in your Rust toolchain is newer than the version supported by your rust-analyzer ({CURRENT_API_VERSION}).
This will prevent proc-macro expansion from working. Please consider updating your rust-analyzer to ensure compatibility with your current toolchain."
),
)),
Ok(v) if v > version::CURRENT_API_VERSION => Err(io::Error::other(
format!( "The version of the proc-macro server ({v}) in your Rust toolchain is newer than the version supported by your rust-analyzer ({}).
This will prevent proc-macro expansion from working. Please consider updating your rust-analyzer to ensure compatibility with your current toolchain."
,version::CURRENT_API_VERSION
),
)),
Ok(v) => {
tracing::info!("Proc-macro server version: {v}");
srv.version = v;
if srv.version >= RUST_ANALYZER_SPAN_SUPPORT {
if let Ok(mode) = srv.enable_rust_analyzer_spans() {
srv.mode = mode;
}
if srv.version >= version::RUST_ANALYZER_SPAN_SUPPORT && let Ok(mode) = srv.enable_rust_analyzer_spans() {
srv.protocol = Protocol::LegacyJson { mode };
}
tracing::info!("Proc-macro server span mode: {:?}", srv.mode);
tracing::info!("Proc-macro server protocol: {:?}", srv.protocol);
Ok(srv)
}
Err(e) => {
@ -98,25 +97,24 @@ impl ProcMacroServerProcess {
self.version
}
/// Enable support for rust-analyzer span mode if the server supports it.
pub(crate) fn rust_analyzer_spans(&self) -> bool {
match self.protocol {
Protocol::LegacyJson { mode } => mode == SpanMode::RustAnalyzer,
}
}
/// Checks the API version of the running proc-macro server.
fn version_check(&self) -> Result<u32, ServerError> {
let request = Request::ApiVersionCheck {};
let response = self.send_task(request)?;
match response {
Response::ApiVersionCheck(version) => Ok(version),
_ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
match self.protocol {
Protocol::LegacyJson { .. } => legacy_protocol::version_check(self),
}
}
/// Enable support for rust-analyzer span mode if the server supports it.
fn enable_rust_analyzer_spans(&self) -> Result<SpanMode, ServerError> {
let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer });
let response = self.send_task(request)?;
match response {
Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode),
_ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
match self.protocol {
Protocol::LegacyJson { .. } => legacy_protocol::enable_rust_analyzer_spans(self),
}
}
@ -125,25 +123,24 @@ impl ProcMacroServerProcess {
&self,
dylib_path: &AbsPath,
) -> Result<Result<Vec<(String, ProcMacroKind)>, String>, ServerError> {
let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() };
let response = self.send_task(request)?;
match response {
Response::ListMacros(it) => Ok(it),
_ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
match self.protocol {
Protocol::LegacyJson { .. } => legacy_protocol::find_proc_macros(self, dylib_path),
}
}
/// Sends a request to the proc-macro server and waits for a response.
pub(crate) fn send_task(&self, req: Request) -> Result<Response, ServerError> {
if let Some(server_error) = self.exited.get() {
return Err(server_error.0.clone());
}
pub(crate) fn send_task<Request, Response>(
&self,
serialize_req: impl FnOnce(
&mut dyn Write,
&mut dyn BufRead,
Request,
&mut String,
) -> Result<Option<Response>, ServerError>,
req: Request,
) -> Result<Response, ServerError> {
let state = &mut *self.state.lock().unwrap();
let mut buf = String::new();
send_request(&mut state.stdin, &mut state.stdout, req, &mut buf)
serialize_req(&mut state.stdin, &mut state.stdout, req, &mut buf)
.and_then(|res| {
res.ok_or_else(|| {
let message = "proc-macro server did not respond with data".to_owned();
@ -162,10 +159,10 @@ impl ProcMacroServerProcess {
Ok(None) | Err(_) => e,
Ok(Some(status)) => {
let mut msg = String::new();
if !status.success() {
if let Some(stderr) = state.process.child.stderr.as_mut() {
_ = stderr.read_to_string(&mut msg);
}
if !status.success()
&& let Some(stderr) = state.process.child.stderr.as_mut()
{
_ = stderr.read_to_string(&mut msg);
}
let server_error = ServerError {
message: format!(
@ -242,21 +239,3 @@ fn mk_child<'a>(
}
cmd.spawn()
}
/// Sends a request to the server and reads the response.
fn send_request(
mut writer: &mut impl Write,
mut reader: &mut impl BufRead,
req: Request,
buf: &mut String,
) -> Result<Option<Response>, ServerError> {
req.write(write_json, &mut writer).map_err(|err| ServerError {
message: "failed to write request".into(),
io: Some(Arc::new(err)),
})?;
let res = Response::read(read_json, &mut reader, buf).map_err(|err| ServerError {
message: "failed to read response".into(),
io: Some(Arc::new(err)),
})?;
Ok(res)
}

1
crates/proc-macro-srv-cli/Cargo.toml
View File

@ -14,6 +14,7 @@ publish = false
proc-macro-srv.workspace = true
proc-macro-api.workspace = true
tt.workspace = true
clap = {version = "4.5.42", default-features = false, features = ["std"]}
[features]
sysroot-abi = ["proc-macro-srv/sysroot-abi"]

39
crates/proc-macro-srv-cli/src/main.rs
View File

@ -9,6 +9,7 @@ extern crate rustc_driver as _;
#[cfg(any(feature = "sysroot-abi", rust_analyzer))]
mod main_loop;
use clap::{Command, ValueEnum};
#[cfg(any(feature = "sysroot-abi", rust_analyzer))]
use main_loop::run;
@ -23,12 +24,46 @@ fn main() -> std::io::Result<()> {
);
std::process::exit(122);
}
let matches = Command::new("proc-macro-srv")
.args(&[clap::Arg::new("format")
.long("format")
.action(clap::ArgAction::Set)
.default_value("json")
.value_parser(clap::builder::EnumValueParser::<ProtocolFormat>::new())])
.get_matches();
let &format =
matches.get_one::<ProtocolFormat>("format").expect("format value should always be present");
run(format)
}
run()
#[derive(Copy, Clone)]
enum ProtocolFormat {
Json,
Postcard,
}
impl ValueEnum for ProtocolFormat {
fn value_variants<'a>() -> &'a [Self] {
&[ProtocolFormat::Json]
}
fn to_possible_value(&self) -> Option<clap::builder::PossibleValue> {
match self {
ProtocolFormat::Json => Some(clap::builder::PossibleValue::new("json")),
ProtocolFormat::Postcard => Some(clap::builder::PossibleValue::new("postcard")),
}
}
fn from_str(input: &str, _ignore_case: bool) -> Result<Self, String> {
match input {
"json" => Ok(ProtocolFormat::Json),
"postcard" => Ok(ProtocolFormat::Postcard),
_ => Err(format!("unknown protocol format: {input}")),
}
}
}
#[cfg(not(any(feature = "sysroot-abi", rust_analyzer)))]
fn run() -> std::io::Result<()> {
fn run(_: ProtocolFormat) -> std::io::Result<()> {
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"proc-macro-srv-cli needs to be compiled with the `sysroot-abi` feature to function"

63
crates/proc-macro-srv-cli/src/main_loop.rs
View File

@ -1,16 +1,47 @@
//! The main loop of the proc-macro server.
use std::io;
use proc_macro_api::legacy_protocol::{
json::{read_json, write_json},
msg::{
self, CURRENT_API_VERSION, ExpandMacroData, ExpnGlobals, Message, SpanMode, TokenId,
deserialize_span_data_index_map, serialize_span_data_index_map,
use proc_macro_api::{
legacy_protocol::{
json::{read_json, write_json},
msg::{
self, ExpandMacroData, ExpnGlobals, Message, SpanMode, SpanTransformer,
deserialize_span_data_index_map, serialize_span_data_index_map,
},
},
version::CURRENT_API_VERSION,
};
use proc_macro_srv::EnvSnapshot;
use proc_macro_srv::{EnvSnapshot, SpanId};
pub(crate) fn run() -> io::Result<()> {
use crate::ProtocolFormat;
struct SpanTrans;
impl SpanTransformer for SpanTrans {
type Table = ();
type Span = SpanId;
fn token_id_of(
_: &mut Self::Table,
span: Self::Span,
) -> proc_macro_api::legacy_protocol::SpanId {
proc_macro_api::legacy_protocol::SpanId(span.0 as u32)
}
fn span_for_token_id(
_: &Self::Table,
id: proc_macro_api::legacy_protocol::SpanId,
) -> Self::Span {
SpanId(id.0 as u32)
}
}
pub(crate) fn run(format: ProtocolFormat) -> io::Result<()> {
match format {
ProtocolFormat::Json => run_json(),
ProtocolFormat::Postcard => unimplemented!(),
}
}
fn run_json() -> io::Result<()> {
fn macro_kind_to_api(kind: proc_macro_srv::ProcMacroKind) -> proc_macro_api::ProcMacroKind {
match kind {
proc_macro_srv::ProcMacroKind::CustomDerive => {
@ -54,13 +85,14 @@ pub(crate) fn run() -> io::Result<()> {
} = *task;
match span_mode {
SpanMode::Id => msg::Response::ExpandMacro({
let def_site = TokenId(def_site as u32);
let call_site = TokenId(call_site as u32);
let mixed_site = TokenId(mixed_site as u32);
let def_site = SpanId(def_site as u32);
let call_site = SpanId(call_site as u32);
let mixed_site = SpanId(mixed_site as u32);
let macro_body = macro_body.to_subtree_unresolved(CURRENT_API_VERSION);
let attributes =
attributes.map(|it| it.to_subtree_unresolved(CURRENT_API_VERSION));
let macro_body =
macro_body.to_subtree_unresolved::<SpanTrans>(CURRENT_API_VERSION);
let attributes = attributes
.map(|it| it.to_subtree_unresolved::<SpanTrans>(CURRENT_API_VERSION));
srv.expand(
lib,
@ -74,7 +106,10 @@ pub(crate) fn run() -> io::Result<()> {
mixed_site,
)
.map(|it| {
msg::FlatTree::new_raw(tt::SubtreeView::new(&it), CURRENT_API_VERSION)
msg::FlatTree::new_raw::<SpanTrans>(
tt::SubtreeView::new(&it),
CURRENT_API_VERSION,
)
})
.map_err(msg::PanicMessage)
}),

8
crates/proc-macro-srv/src/lib.rs
View File

@ -41,10 +41,12 @@ use std::{
};
use paths::{Utf8Path, Utf8PathBuf};
use span::{Span, TokenId};
use span::Span;
use crate::server_impl::TokenStream;
pub use crate::server_impl::token_id::SpanId;
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
pub enum ProcMacroKind {
CustomDerive,
@ -159,8 +161,8 @@ pub trait ProcMacroSrvSpan: Copy + Send {
fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server;
}
impl ProcMacroSrvSpan for TokenId {
type Server = server_impl::token_id::TokenIdServer;
impl ProcMacroSrvSpan for SpanId {
type Server = server_impl::token_id::SpanIdServer;
fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server {
Self::Server { call_site, def_site, mixed_site }

53
crates/proc-macro-srv/src/server_impl/token_id.rs
View File

@ -1,4 +1,4 @@
//! proc-macro server backend based on [`proc_macro_api::msg::TokenId`] as the backing span.
//! proc-macro server backend based on [`proc_macro_api::msg::SpanId`] as the backing span.
//! This backend is rather inflexible, used by RustRover and older rust-analyzer versions.
use std::ops::{Bound, Range};
@ -7,25 +7,34 @@ use proc_macro::bridge::{self, server};
use crate::server_impl::{from_token_tree, literal_from_str, token_stream::TokenStreamBuilder};
type Span = span::TokenId;
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct SpanId(pub u32);
impl std::fmt::Debug for SpanId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
type Span = SpanId;
type TokenStream = crate::server_impl::TokenStream<Span>;
pub struct FreeFunctions;
pub struct TokenIdServer {
pub struct SpanIdServer {
pub call_site: Span,
pub def_site: Span,
pub mixed_site: Span,
}
impl server::Types for TokenIdServer {
impl server::Types for SpanIdServer {
type FreeFunctions = FreeFunctions;
type TokenStream = TokenStream;
type Span = Span;
type Symbol = Symbol;
}
impl server::FreeFunctions for TokenIdServer {
impl server::FreeFunctions for SpanIdServer {
fn injected_env_var(&mut self, _: &str) -> Option<std::string::String> {
None
}
@ -41,7 +50,7 @@ impl server::FreeFunctions for TokenIdServer {
fn emit_diagnostic(&mut self, _: bridge::Diagnostic<Self::Span>) {}
}
impl server::TokenStream for TokenIdServer {
impl server::TokenStream for SpanIdServer {
fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
stream.is_empty()
}
@ -102,12 +111,12 @@ impl server::TokenStream for TokenIdServer {
&mut self,
stream: Self::TokenStream,
) -> Vec<bridge::TokenTree<Self::TokenStream, Self::Span, Self::Symbol>> {
// Can't join with `TokenId`.
// Can't join with `SpanId`.
stream.into_bridge(&mut |first, _second| first)
}
}
impl server::Span for TokenIdServer {
impl server::Span for SpanIdServer {
fn debug(&mut self, span: Self::Span) -> String {
format!("{:?}", span.0)
}
@ -174,14 +183,14 @@ impl server::Span for TokenIdServer {
}
}
impl server::Symbol for TokenIdServer {
impl server::Symbol for SpanIdServer {
fn normalize_and_validate_ident(&mut self, string: &str) -> Result<Self::Symbol, ()> {
// FIXME: nfc-normalize and validate idents
Ok(<Self as server::Server>::intern_symbol(string))
}
}
impl server::Server for TokenIdServer {
impl server::Server for SpanIdServer {
fn globals(&mut self) -> bridge::ExpnGlobals<Self::Span> {
bridge::ExpnGlobals {
def_site: self.def_site,
@ -201,8 +210,6 @@ impl server::Server for TokenIdServer {
#[cfg(test)]
mod tests {
use span::TokenId;
use super::*;
#[test]
@ -211,18 +218,18 @@ mod tests {
token_trees: vec![
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
sym: Symbol::intern("struct"),
span: TokenId(0),
span: SpanId(0),
is_raw: tt::IdentIsRaw::No,
})),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
sym: Symbol::intern("T"),
span: TokenId(0),
span: SpanId(0),
is_raw: tt::IdentIsRaw::No,
})),
tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter {
open: TokenId(0),
close: TokenId(0),
open: SpanId(0),
close: SpanId(0),
kind: tt::DelimiterKind::Brace,
},
len: 0,
@ -238,8 +245,8 @@ mod tests {
let subtree_paren_a = vec![
tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter {
open: TokenId(0),
close: TokenId(0),
open: SpanId(0),
close: SpanId(0),
kind: tt::DelimiterKind::Parenthesis,
},
len: 1,
@ -247,24 +254,24 @@ mod tests {
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
is_raw: tt::IdentIsRaw::No,
sym: Symbol::intern("a"),
span: TokenId(0),
span: SpanId(0),
})),
];
let t1 = TokenStream::from_str("(a)", TokenId(0)).unwrap();
let t1 = TokenStream::from_str("(a)", SpanId(0)).unwrap();
assert_eq!(t1.token_trees.len(), 2);
assert!(t1.token_trees[0..2] == subtree_paren_a);
let t2 = TokenStream::from_str("(a);", TokenId(0)).unwrap();
let t2 = TokenStream::from_str("(a);", SpanId(0)).unwrap();
assert_eq!(t2.token_trees.len(), 3);
assert!(t2.token_trees[0..2] == subtree_paren_a);
let underscore = TokenStream::from_str("_", TokenId(0)).unwrap();
let underscore = TokenStream::from_str("_", SpanId(0)).unwrap();
assert!(
underscore.token_trees[0]
== tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
sym: Symbol::intern("_"),
span: TokenId(0),
span: SpanId(0),
is_raw: tt::IdentIsRaw::No,
}))
);

14
crates/proc-macro-srv/src/tests/utils.rs
View File

@ -1,14 +1,12 @@
//! utils used in proc-macro tests
use expect_test::Expect;
use span::{
EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TokenId,
};
use span::{EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext};
use tt::TextRange;
use crate::{EnvSnapshot, ProcMacroSrv, dylib, proc_macro_test_dylib_path};
use crate::{EnvSnapshot, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path};
fn parse_string(call_site: TokenId, src: &str) -> crate::server_impl::TokenStream<TokenId> {
fn parse_string(call_site: SpanId, src: &str) -> crate::server_impl::TokenStream<SpanId> {
crate::server_impl::TokenStream::with_subtree(crate::server_impl::TopSubtree(
syntax_bridge::parse_to_token_tree_static_span(span::Edition::CURRENT, call_site, src)
.unwrap()
@ -59,9 +57,9 @@ fn assert_expand_impl(
let path = proc_macro_test_dylib_path();
let expander = dylib::Expander::new(&path).unwrap();
let def_site = TokenId(0);
let call_site = TokenId(1);
let mixed_site = TokenId(2);
let def_site = SpanId(0);
let call_site = SpanId(1);
let mixed_site = SpanId(2);
let input_ts = parse_string(call_site, input).into_subtree(call_site);
let attr_ts = attr.map(|attr| parse_string(call_site, attr).into_subtree(call_site));
let input_ts_string = format!("{input_ts:?}");

12
crates/span/src/lib.rs
View File

@ -203,15 +203,3 @@ pub struct HirFileId(pub salsa::Id);
/// `println!("Hello, {}", world)`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MacroCallId(pub salsa::Id);
/// Legacy span type, only defined here as it is still used by the proc-macro server.
/// While rust-analyzer doesn't use this anymore at all, RustRover relies on the legacy type for
/// proc-macro expansion.
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct TokenId(pub u32);
impl std::fmt::Debug for TokenId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}