├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── nix-errors ├── Cargo.toml └── src │ └── lib.rs ├── nix-lexer ├── Cargo.toml ├── benches │ └── lexer.rs ├── examples │ └── tokenize.rs ├── src │ ├── lib.rs │ ├── split.rs │ ├── tokens.rs │ └── unescape.rs └── tests │ ├── literals.nix │ ├── literals.snap │ ├── operators.nix │ ├── operators.snap │ ├── punctuation.nix │ ├── punctuation.snap │ ├── snapshots.rs │ ├── trivia.nix │ └── trivia.snap ├── nix-parser ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── benches │ └── example.rs ├── example.nix ├── examples │ └── viewer.rs ├── grammar.abnf └── src │ ├── ast.rs │ ├── ast │ ├── fmt.rs │ ├── macros.rs │ └── tokens.rs │ ├── error.rs │ ├── error │ ├── expected_found.rs │ ├── incorrect_delim.rs │ ├── unclosed_delim.rs │ └── unexpected.rs │ ├── lexer.rs │ ├── lexer │ ├── lexers.rs │ ├── lexers │ │ ├── number.rs │ │ ├── path.rs │ │ ├── string.rs │ │ └── uri.rs │ ├── tokens.rs │ └── util.rs │ ├── lib.rs │ ├── parser.rs │ └── parser │ ├── expr.rs │ ├── expr │ ├── atomic.rs │ ├── attr.rs │ ├── bind.rs │ ├── func.rs │ ├── stmt.rs │ └── util.rs │ ├── partial.rs │ └── tokens.rs ├── nix-parser2 ├── Cargo.toml └── src │ ├── ast.rs │ ├── cst.rs │ ├── error.rs │ ├── lexer.rs │ └── lib.rs ├── shell.nix └── src ├── backend.rs ├── lib.rs └── main.rs /.gitignore: -------------------------------------------------------------------------------- 1 | **/target 2 | **/*.rs.bk 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nix-language-server" 3 | version = "0.1.0" 4 | authors = ["Eyal Kalderon "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | codespan = "0.8" 9 | codespan-lsp = "0.8" 10 | env_logger = "0.7" 11 | jsonrpc-core = "14.0" 12 | log = "0.4" 13 | nix-parser = { version = "0.1.0", path = "./nix-parser" } 14 | nix-parser2 = { version = "0.1.0", path = "./nix-parser2", default-features = false } 15 | structopt = "0.3" 16 | tokio = { version = "0.2", features = ["full"] } 17 | tower-lsp = "0.9.0" 18 | 19 | [profile.release] 20 | codegen-units = 1 21 | lto = true 22 | 23 | [workspace] 24 | members = ["nix-parser", "nix-errors", "nix-lexer", "nix-parser2"] 25 | -------------------------------------------------------------------------------- /nix-errors/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nix-errors" 3 | version = "0.1.0" 4 | authors = ["Eyal Kalderon "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | codespan = "0.9.3" 9 | codespan-lsp = "0.9.3" 10 | codespan-reporting = "0.9.3" 11 | lsp-types = "0.74" 12 | smallvec = "1.2" 13 | -------------------------------------------------------------------------------- /nix-errors/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Common types and traits used by all kinds of errors. 2 | 3 | use std::fmt::{self, Display, Formatter}; 4 | use std::iter::FromIterator; 5 | use std::slice::Iter; 6 | 7 | use codespan::{FileId, Files}; 8 | use codespan_reporting::diagnostic::Diagnostic; 9 | use lsp_types::Diagnostic as LspDiagnostic; 10 | use smallvec::SmallVec; 11 | 12 | /// Number of errors to keep on the stack before spilling onto the heap. 13 | const NUM_ERRORS: usize = 4; 14 | 15 | /// A specialized `Result` type for LSP diagnostic conversions. 16 | pub type LspResult = std::result::Result; 17 | 18 | /// A trait for converting an error type into a reportable diagnostic. 19 | /// 20 | /// This trait is generic so that both CLI diagnostics and Language Server diagnostics can be 21 | /// produced using this interface. 22 | pub trait ToDiagnostic { 23 | /// Converts the error to a diagnostic `D` for the given source file specified by `file_id`. 24 | fn to_diagnostic>(&self, files: &Files, file_id: FileId) -> D; 25 | } 26 | 27 | /// A generic growable stack for accumulating errors. 28 | #[derive(Clone, Debug, PartialEq)] 29 | pub struct Errors(SmallVec<[E; NUM_ERRORS]>); 30 | 31 | impl Errors { 32 | /// Constructs a new, empty `Errors` stack. 33 | /// 34 | /// The stack will not allocate until new errors are pushed onto it. 35 | #[inline] 36 | pub fn new() -> Self { 37 | Errors(SmallVec::new()) 38 | } 39 | 40 | /// Returns the number of errors in the stack. 41 | #[inline] 42 | pub fn len(&self) -> usize { 43 | self.0.len() 44 | } 45 | 46 | /// Returns `true` if the error stack is empty. 47 | #[inline] 48 | pub fn is_empty(&self) -> bool { 49 | self.0.is_empty() 50 | } 51 | 52 | /// Appends a new error to the stack. 53 | pub fn push(&mut self, error: E) { 54 | self.0.push(error); 55 | } 56 | 57 | /// Removes the last error from the stack and returns it, or [`None`] if it is empty. 58 | /// 59 | /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None 60 | pub fn pop(&mut self) -> Option { 61 | self.0.pop() 62 | } 63 | 64 | /// Returns the last error in the stack, or [`None`] if it is empty. 65 | /// 66 | /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None 67 | pub fn last(&self) -> Option<&E> { 68 | self.0.last() 69 | } 70 | 71 | /// Returns an iterator of errors. 72 | pub fn iter(&self) -> Iter { 73 | self.0.iter() 74 | } 75 | } 76 | 77 | impl Errors 78 | where 79 | E: ToDiagnostic>, 80 | { 81 | /// Returns an iterator which yields each error converted to a [`Diagnostic`]. 82 | /// 83 | /// [`Diagnostic`]: https://docs.rs/codespan-reporting/0.9.1/codespan_reporting/diagnostic/struct.Diagnostic.html 84 | pub fn to_diagnostics<'a, S>( 85 | &'a self, 86 | files: &'a Files, 87 | file_id: FileId, 88 | ) -> impl Iterator> + 'a 89 | where 90 | S: AsRef, 91 | { 92 | self.iter().map(move |e| e.to_diagnostic(files, file_id)) 93 | } 94 | } 95 | 96 | impl Errors 97 | where 98 | E: ToDiagnostic>, 99 | { 100 | /// Returns an iterator which yields each error converted to an LSP [`Diagnostic`]. 101 | /// 102 | /// [`Diagnostic`]: https://docs.rs/lsp-types/0.73.0/lsp_types/struct.Diagnostic.html 103 | pub fn to_lsp_diagnostics<'a, S>( 104 | &'a self, 105 | files: &'a Files, 106 | file_id: FileId, 107 | ) -> impl Iterator> + 'a 108 | where 109 | S: AsRef, 110 | { 111 | self.iter().map(move |e| e.to_diagnostic(files, file_id)) 112 | } 113 | } 114 | 115 | impl Default for Errors { 116 | /// Creates an empty `Errors` stack. 117 | fn default() -> Self { 118 | Errors::new() 119 | } 120 | } 121 | 122 | impl Extend for Errors { 123 | fn extend>(&mut self, iter: I) { 124 | self.0.extend(iter) 125 | } 126 | } 127 | 128 | impl FromIterator for Errors { 129 | fn from_iter>(iter: I) -> Self { 130 | Errors(SmallVec::from_iter(iter)) 131 | } 132 | } 133 | 134 | impl IntoIterator for Errors { 135 | type Item = E; 136 | type IntoIter = smallvec::IntoIter<[E; NUM_ERRORS]>; 137 | 138 | fn into_iter(self) -> Self::IntoIter { 139 | self.0.into_iter() 140 | } 141 | } 142 | 143 | impl<'a, E> IntoIterator for &'a Errors { 144 | type Item = &'a E; 145 | type IntoIter = Iter<'a, E>; 146 | 147 | fn into_iter(self) -> Self::IntoIter { 148 | self.iter() 149 | } 150 | } 151 | 152 | /// Helper struct for producing pretty "expected foo, found bar" error messages. 153 | #[derive(Clone, Debug, Eq, PartialEq)] 154 | pub struct ExpectedFound { 155 | expected: SmallVec<[T; 4]>, 156 | found: U, 157 | } 158 | 159 | impl ExpectedFound 160 | where 161 | T: Display, 162 | U: Display, 163 | { 164 | /// Creates a new `ExpectedFound` from the given list of expected items and a found item. 165 | pub fn new(expected: impl IntoIterator, found: U) -> Self { 166 | ExpectedFound { 167 | expected: expected.into_iter().collect(), 168 | found, 169 | } 170 | } 171 | 172 | /// Returns only the "expected" part of the error message string. 173 | /// 174 | /// This message is useful for diagnostic labels which annotate the source text. 175 | pub fn expected_message(&self) -> String { 176 | match self.expected.as_slice() { 177 | [single] => format!("expected {}", single), 178 | [first, second] => format!("expected {} or {}", first, second), 179 | [first @ .., last] => { 180 | let first: Vec<_> = first.iter().map(ToString::to_string).collect(); 181 | format!("expected one of {} or {}", first.join(", "), last) 182 | } 183 | [] => "expected ".to_string(), 184 | } 185 | } 186 | } 187 | 188 | /// Displays the complete error message string. 189 | impl Display for ExpectedFound 190 | where 191 | T: Display, 192 | U: Display, 193 | { 194 | fn fmt(&self, f: &mut Formatter) -> fmt::Result { 195 | write!(f, "{}, found {}", self.expected_message(), self.found) 196 | } 197 | } 198 | 199 | /// A partial value which might contain errors. 200 | /// 201 | /// `Partial` is a tuple-like data structure which contains some value and an associated 202 | /// [`Errors`] stack. 203 | /// 204 | /// [`Errors`]: ../struct.Errors.html 205 | /// 206 | /// This type is used to accumulate errors and apply monadic transformations to possibly incomplete 207 | /// or invalid data structures. Consumers of `Partial` values can choose to assert that the 208 | /// contained value exists without errors with [`Partial::verify()`], which will consume the 209 | /// `Partial` and transform it into a `Result>` which can be handled normally. 210 | /// 211 | /// [`Partial::verify()`]: #method.verify 212 | #[must_use = "partial values must be either verified or destructured"] 213 | #[derive(Clone, Debug, PartialEq)] 214 | pub struct Partial { 215 | value: T, 216 | errors: Errors, 217 | } 218 | 219 | impl Partial { 220 | /// Constructs a new `Partial` with the given initial value. 221 | pub fn new(value: T) -> Self { 222 | Partial::with_errors(value, Errors::new()) 223 | } 224 | 225 | /// Constructs a new `Partial` with the given initial value and a stack of errors. 226 | pub fn with_errors(value: T, errors: Errors) -> Self { 227 | Partial { value, errors } 228 | } 229 | 230 | /// Returns whether this partial value contains errors. 231 | pub fn has_errors(&self) -> bool { 232 | !self.errors.is_empty() 233 | } 234 | 235 | /// Returns the contained partial value, if any. 236 | pub fn value(&self) -> &T { 237 | &self.value 238 | } 239 | 240 | /// Returns a reference to any errors associated with the partial value. 241 | pub fn errors(&self) -> &Errors { 242 | &self.errors 243 | } 244 | 245 | /// Appends the given error to the error stack contained in this partial value. 246 | pub fn extend_errors>(&mut self, error: I) { 247 | self.errors.extend(error); 248 | } 249 | 250 | /// Calls `f` on the contained value and accumulates any errors it may have produced. 251 | pub fn flat_map(mut self, f: F) -> Partial 252 | where 253 | F: FnOnce(T) -> Partial, 254 | { 255 | let mut partial = f(self.value); 256 | self.errors.extend(partial.errors); 257 | partial.errors = self.errors; 258 | partial 259 | } 260 | 261 | /// Destructures this `Partial` into a tuple of its inner components. 262 | pub fn into_inner(self) -> (T, Errors) { 263 | (self.value, self.errors) 264 | } 265 | 266 | /// Transforms the `Partial` into a `Result>`, asserting that the contained 267 | /// value has no errors. 268 | pub fn verify(self) -> Result> { 269 | if self.errors.is_empty() { 270 | Ok(self.value) 271 | } else { 272 | Err(self.errors) 273 | } 274 | } 275 | } 276 | -------------------------------------------------------------------------------- /nix-lexer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nix-lexer" 3 | version = "0.1.0" 4 | authors = ["Eyal Kalderon "] 5 | edition = "2018" 6 | 7 | [features] 8 | default = [] 9 | serialization = ["codespan/serialization", "serde"] 10 | 11 | [dependencies] 12 | codespan = "0.9" 13 | nom = { version = "5.1", default-features = false } 14 | nom_locate = "2.0" 15 | smallvec = "1.2" 16 | 17 | serde = { version = "1.0", features = ["derive"], optional = true } 18 | 19 | [dev-dependencies] 20 | criterion = "0.3" 21 | nix-parser = { path = "../nix-parser" } 22 | serde_json = "1.0" 23 | 24 | [[bench]] 25 | name = "lexer" 26 | harness = false 27 | -------------------------------------------------------------------------------- /nix-lexer/benches/lexer.rs: -------------------------------------------------------------------------------- 1 | use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; 2 | use nix_parser::lexer::Lexer; 3 | 4 | const EXAMPLE_FILE: &str = include_str!(concat!( 5 | env!("CARGO_MANIFEST_DIR"), 6 | "/../nix-parser/example.nix" 7 | )); 8 | 9 | fn lexer(b: &mut Criterion) { 10 | let mut group = b.benchmark_group("lexer"); 11 | 12 | let module = black_box(EXAMPLE_FILE); 13 | group.throughput(Throughput::Bytes(EXAMPLE_FILE.len() as u64)); 14 | group.bench_function("old", move |b| { 15 | b.iter(|| { 16 | let lexer = Lexer::new_with_whitespace(module).unwrap(); 17 | black_box(lexer); 18 | }); 19 | }); 20 | group.bench_function("new", move |b| { 21 | b.iter(|| { 22 | let lexer: Vec<_> = nix_lexer::tokenize(module).collect(); 23 | black_box(lexer); 24 | }); 25 | }); 26 | 27 | group.finish() 28 | } 29 | 30 | criterion_group!(benches, lexer); 31 | criterion_main!(benches); 32 | -------------------------------------------------------------------------------- /nix-lexer/examples/tokenize.rs: -------------------------------------------------------------------------------- 1 | use std::io::Read; 2 | 3 | use codespan::Files; 4 | 5 | fn main() { 6 | let mut buffer = String::new(); 7 | std::io::stdin().read_to_string(&mut buffer).unwrap(); 8 | 9 | let mut files = Files::new(); 10 | let file_id = files.add("", &buffer); 11 | let source = files.source(file_id); 12 | 13 | for token in nix_lexer::tokenize(&source) { 14 | println!("{}", token.display(&source)); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /nix-lexer/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Low-level lexer for the Nix language. 2 | 3 | #![forbid(unsafe_code)] 4 | 5 | pub use self::split::split_lines_without_indent; 6 | pub use self::tokens::{DisplayToken, LiteralKind, StringKind, Token, TokenKind}; 7 | pub use self::unescape::unescape_str; 8 | 9 | use codespan::Span; 10 | use nom::branch::alt; 11 | use nom::bytes::complete::{is_a, tag, take_until, take_while, take_while1}; 12 | use nom::character::complete::{ 13 | anychar, char, digit0, digit1, multispace1, none_of, not_line_ending, one_of, 14 | }; 15 | use nom::combinator::{map, opt, peek, recognize, verify}; 16 | use nom::error::ErrorKind; 17 | use nom::multi::{many0_count, many1_count, many_till}; 18 | use nom::sequence::{delimited, pair, preceded, terminated, tuple}; 19 | use smallvec::SmallVec; 20 | 21 | mod split; 22 | mod tokens; 23 | mod unescape; 24 | 25 | type LocatedSpan<'a> = nom_locate::LocatedSpan<&'a str>; 26 | type IResult<'a, T> = nom::IResult, T>; 27 | 28 | trait ToSpan { 29 | fn to_span(&self) -> Span; 30 | } 31 | 32 | impl<'a> ToSpan for LocatedSpan<'a> { 33 | fn to_span(&self) -> Span { 34 | let start = self.location_offset(); 35 | let end = start + self.fragment().len(); 36 | Span::new(start as u32, end as u32) 37 | } 38 | } 39 | 40 | /// A list of all possible lexer modes. 41 | #[derive(Clone, Copy)] 42 | enum Mode { 43 | /// Default lexer mode. 44 | Normal, 45 | /// Mode used when tokenizing a string. 46 | String(StringKind), 47 | } 48 | 49 | struct LexerModes(SmallVec<[Mode; 8]>); 50 | 51 | impl LexerModes { 52 | fn new() -> Self { 53 | LexerModes(SmallVec::with_capacity(8)) 54 | } 55 | 56 | fn current(&self) -> &Mode { 57 | self.0.last().unwrap_or(&Mode::Normal) 58 | } 59 | 60 | fn push(&mut self, mode: Mode) { 61 | self.0.push(mode) 62 | } 63 | 64 | fn pop(&mut self) { 65 | self.0.pop(); 66 | } 67 | } 68 | 69 | /// Converts an input string into a sequence of tokens. 70 | pub fn tokenize(input: &str) -> impl Iterator + '_ { 71 | let mut input = LocatedSpan::new(input); 72 | let mut modes = LexerModes::new(); 73 | std::iter::from_fn(move || { 74 | let (remaining, out) = next_token(input, *modes.current())?; 75 | input = remaining; 76 | 77 | match (modes.current(), out.kind) { 78 | // Switch to and from `Normal` and `String` modes when encountering `"` or `''`. 79 | (Mode::Normal, TokenKind::StringTerm { kind }) => modes.push(Mode::String(kind)), 80 | (Mode::String(kind), TokenKind::StringTerm { kind: term_kind }) => { 81 | debug_assert_eq!(*kind, term_kind); 82 | modes.pop(); 83 | } 84 | 85 | // Switch back to `Normal` mode when a string interpolation is detected. 86 | (Mode::String(_), TokenKind::Interpolate) => modes.push(Mode::Normal), 87 | 88 | // Count opening and closing braces, popping back to the previous mode, if any. 89 | (Mode::Normal, TokenKind::Interpolate) => modes.push(Mode::Normal), 90 | (Mode::Normal, TokenKind::OpenBrace) => modes.push(Mode::Normal), 91 | (Mode::Normal, TokenKind::CloseBrace) => modes.pop(), 92 | 93 | _ => (), 94 | } 95 | 96 | Some(out) 97 | }) 98 | } 99 | 100 | fn next_token(input: LocatedSpan, mode: Mode) -> Option<(LocatedSpan, Token)> { 101 | let result = match mode { 102 | Mode::Normal => alt(( 103 | line_comment, 104 | block_comment, 105 | whitespace, 106 | path, 107 | uri, 108 | identifier, 109 | float, 110 | integer, 111 | path_template, 112 | interpolate, 113 | symbol, 114 | string_term, 115 | unknown, 116 | ))(input), 117 | Mode::String(_) => alt((interpolate, string_literal(mode), string_term))(input), 118 | }; 119 | 120 | match result { 121 | Ok((remaining, token)) => Some((remaining, token)), 122 | Err(nom::Err::Error(_)) => None, 123 | Err(nom::Err::Failure(err)) => unreachable!("lexer failed with: {:?}", err), 124 | Err(nom::Err::Incomplete(inc)) => unreachable!("lexer returned incomplete: {:?}", inc), 125 | } 126 | } 127 | 128 | fn line_comment(input: LocatedSpan) -> IResult { 129 | let comment = recognize(preceded(char('#'), not_line_ending)); 130 | map(comment, |span: LocatedSpan| { 131 | Token::new(TokenKind::LineComment, span.to_span()) 132 | })(input) 133 | } 134 | 135 | fn block_comment(input: LocatedSpan) -> IResult { 136 | let (remaining, span, terminated) = match recognize(pair(tag("/*"), take_until("*/")))(input) { 137 | Ok((remaining, span)) => (remaining, span, true), 138 | Err(nom::Err::Error((remaining, ErrorKind::TakeUntil))) => (remaining, input, false), 139 | Err(err) => return Err(err), 140 | }; 141 | 142 | let token = Token::new(TokenKind::BlockComment { terminated }, span.to_span()); 143 | Ok((remaining, token)) 144 | } 145 | 146 | fn whitespace(input: LocatedSpan) -> IResult { 147 | map(multispace1, |span: LocatedSpan| { 148 | Token::new(TokenKind::Whitespace, span.to_span()) 149 | })(input) 150 | } 151 | 152 | fn identifier(input: LocatedSpan) -> IResult { 153 | let first = verify(anychar, |c| c.is_ascii_alphabetic() || *c == '_'); 154 | let rest = take_while(|c: char| c.is_ascii_alphanumeric() || "_-'".contains(c)); 155 | let ident = recognize(preceded(first, rest)); 156 | map(ident, |span: LocatedSpan| { 157 | Token::new(TokenKind::Ident, span.to_span()) 158 | })(input) 159 | } 160 | 161 | fn is_path_segment(c: char) -> bool { 162 | c.is_ascii_alphanumeric() || "._-+".contains(c) 163 | } 164 | 165 | fn path(input: LocatedSpan) -> IResult { 166 | let segments = many1_count(preceded(char('/'), take_while1(is_path_segment))); 167 | let relative = map(pair(take_while(is_path_segment), &segments), |_| ()); 168 | let home = map(pair(char('~'), &segments), |_| ()); 169 | let (remaining, span) = recognize(pair(alt((relative, home)), opt(char('/'))))(input)?; 170 | 171 | let trailing_slash = span.fragment().ends_with('/'); 172 | let token_kind = TokenKind::Literal { 173 | kind: LiteralKind::Path { trailing_slash }, 174 | }; 175 | 176 | Ok((remaining, Token::new(token_kind, span.to_span()))) 177 | } 178 | 179 | fn uri(input: LocatedSpan) -> IResult { 180 | let first = verify(anychar, |c| c.is_ascii_alphabetic()); 181 | let rest = take_while(|c: char| c.is_ascii_alphanumeric() || "+-.".contains(c)); 182 | let scheme = preceded(first, rest); 183 | 184 | let path = take_while1(|c: char| c.is_ascii_alphanumeric() || "%/?:@&=+$,-_.!~*'".contains(c)); 185 | let uri = recognize(tuple((scheme, char(':'), path))); 186 | 187 | map(uri, |span: LocatedSpan| { 188 | let kind = LiteralKind::Uri; 189 | let token_kind = TokenKind::Literal { kind }; 190 | Token::new(token_kind, span.to_span()) 191 | })(input) 192 | } 193 | 194 | fn float(input: LocatedSpan) -> IResult { 195 | let first = preceded(is_a("123456789"), digit0); 196 | let positive = map(tuple((first, char('.'), digit0)), |_| ()); 197 | let fraction = map(tuple((opt(char('0')), char('.'), digit1)), |_| ()); 198 | let exp = tuple((one_of("Ee"), opt(one_of("+-")), digit1)); 199 | let float = recognize(preceded(alt((positive, fraction)), opt(exp))); 200 | 201 | map(float, |span: LocatedSpan| { 202 | let kind = LiteralKind::Float; 203 | let token_kind = TokenKind::Literal { kind }; 204 | Token::new(token_kind, span.to_span()) 205 | })(input) 206 | } 207 | 208 | fn integer(input: LocatedSpan) -> IResult { 209 | map(digit1, |span: LocatedSpan| { 210 | let kind = LiteralKind::Integer; 211 | let token_kind = TokenKind::Literal { kind }; 212 | Token::new(token_kind, span.to_span()) 213 | })(input) 214 | } 215 | 216 | fn path_template(input: LocatedSpan) -> IResult { 217 | let segment = take_while1(is_path_segment); 218 | let segments = terminated(&segment, many0_count(preceded(char('/'), &segment))); 219 | let path = terminated(segments, opt(char('/'))); 220 | let (remaining, span) = recognize(delimited(char('<'), path, char('>')))(input)?; 221 | 222 | let trailing_slash = span.fragment().ends_with("/>"); 223 | let token_kind = TokenKind::Literal { 224 | kind: LiteralKind::PathTemplate { trailing_slash }, 225 | }; 226 | 227 | Ok((remaining, Token::new(token_kind, span.to_span()))) 228 | } 229 | 230 | fn interpolate(input: LocatedSpan) -> IResult { 231 | map(tag("${"), |span: LocatedSpan| { 232 | Token::new(TokenKind::Interpolate, span.to_span()) 233 | })(input) 234 | } 235 | 236 | fn symbol(input: LocatedSpan) -> IResult { 237 | let punctuation = alt(( 238 | map(tag(";"), |span: LocatedSpan| (TokenKind::Semi, span)), 239 | map(tag(","), |span| (TokenKind::Comma, span)), 240 | map(tag("..."), |span| (TokenKind::Ellipsis, span)), 241 | map(tag("."), |span| (TokenKind::Dot, span)), 242 | map(tag("{"), |span| (TokenKind::OpenBrace, span)), 243 | map(tag("}"), |span| (TokenKind::CloseBrace, span)), 244 | map(tag("("), |span| (TokenKind::OpenParen, span)), 245 | map(tag(")"), |span| (TokenKind::CloseParen, span)), 246 | map(tag("["), |span| (TokenKind::OpenBracket, span)), 247 | map(tag("]"), |span| (TokenKind::CloseBracket, span)), 248 | map(tag("?"), |span| (TokenKind::Question, span)), 249 | map(tag(":"), |span| (TokenKind::Colon, span)), 250 | map(tag("@"), |span| (TokenKind::At, span)), 251 | )); 252 | 253 | let token = alt(( 254 | punctuation, 255 | map(tag("=="), |span| (TokenKind::IsEq, span)), 256 | map(tag("="), |span| (TokenKind::Eq, span)), 257 | map(tag("++"), |span| (TokenKind::Concat, span)), 258 | map(tag("+"), |span| (TokenKind::Add, span)), 259 | map(tag("->"), |span| (TokenKind::Imply, span)), 260 | map(tag("-"), |span| (TokenKind::Sub, span)), 261 | map(tag("*"), |span| (TokenKind::Mul, span)), 262 | map(tag("//"), |span| (TokenKind::Update, span)), 263 | map(tag("/"), |span| (TokenKind::Div, span)), 264 | map(tag("!="), |span| (TokenKind::NotEq, span)), 265 | map(tag("!"), |span| (TokenKind::Not, span)), 266 | map(tag("<="), |span| (TokenKind::LessThanEq, span)), 267 | map(tag("<"), |span| (TokenKind::LessThan, span)), 268 | map(tag(">="), |span| (TokenKind::GreaterThanEq, span)), 269 | map(tag(">"), |span| (TokenKind::GreaterThan, span)), 270 | map(tag("&&"), |span| (TokenKind::LogicalAnd, span)), 271 | map(tag("||"), |span| (TokenKind::LogicalOr, span)), 272 | )); 273 | 274 | map(token, |(kind, span)| Token::new(kind, span.to_span()))(input) 275 | } 276 | 277 | fn unknown(input: LocatedSpan) -> IResult { 278 | map(recognize(anychar), |span: LocatedSpan| { 279 | Token::new(TokenKind::Unknown, span.to_span()) 280 | })(input) 281 | } 282 | 283 | fn string_term(input: LocatedSpan) -> IResult { 284 | let normal = map(tag("\""), |span: LocatedSpan| (StringKind::Normal, span)); 285 | let indented = map(tag("''"), |span: LocatedSpan| (StringKind::Indented, span)); 286 | let term = alt((normal, indented)); 287 | 288 | map(term, |(kind, span)| { 289 | Token::new(TokenKind::StringTerm { kind }, span.to_span()) 290 | })(input) 291 | } 292 | 293 | fn string_literal<'a>(mode: Mode) -> impl Fn(LocatedSpan<'a>) -> IResult { 294 | fn many_till_ne<'a, F, G>(f: F, term: G) -> impl Fn(LocatedSpan<'a>) -> IResult 295 | where 296 | F: Fn(LocatedSpan<'a>) -> IResult, 297 | G: Fn(LocatedSpan<'a>) -> IResult, 298 | { 299 | move |i| { 300 | if peek(&term)(i).is_ok() { 301 | return Err(nom::Err::Error((i, ErrorKind::NonEmpty))); 302 | } 303 | 304 | let literal_chunk = alt((&f, recognize(anychar))); 305 | match recognize(many_till(literal_chunk, peek(&term)))(i) { 306 | Ok((remaining, span)) => Ok((remaining, span)), 307 | Err(nom::Err::Error((remaining, ErrorKind::Eof))) if !i.fragment().is_empty() => { 308 | Ok((remaining, i)) 309 | } 310 | Err(err) => Err(err), 311 | } 312 | } 313 | } 314 | 315 | move |input| { 316 | let (remaining, span) = match mode { 317 | Mode::String(StringKind::Normal) => { 318 | let escape = terminated(tag("\\"), one_of("\"$")); 319 | let end_tag = tag("\""); 320 | many_till_ne(escape, alt((end_tag, recognize(interpolate))))(input)? 321 | } 322 | Mode::String(StringKind::Indented) => { 323 | let escape = terminated(tag("''"), one_of("'$")); 324 | let end_tag = terminated(tag("''"), peek(none_of("'$"))); 325 | many_till_ne(escape, alt((end_tag, recognize(interpolate))))(input)? 326 | } 327 | _ => panic!("string_literal() called outside String lexer mode"), 328 | }; 329 | 330 | let token_kind = TokenKind::StringLiteral; 331 | Ok((remaining, Token::new(token_kind, span.to_span()))) 332 | } 333 | } 334 | -------------------------------------------------------------------------------- /nix-lexer/src/split.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for splitting multi-line string literals and block comments. 2 | 3 | use nom::character::complete::multispace0; 4 | 5 | use super::LocatedSpan; 6 | 7 | /// Splits the input string into lines with leading indentation normalized appropriately. 8 | /// 9 | /// This utility function is useful for splitting multi-line `''` string literals or block comments 10 | /// in order to retrieve their actual text while preserving meaningful indentation inside of them. 11 | /// 12 | /// # Examples 13 | /// 14 | /// ``` 15 | /// use nix_lexer::split_lines_without_indent; 16 | /// 17 | /// let string_literal = " 18 | /// hello world 19 | /// 20 | /// * bullet 1 21 | /// * bullet 2 22 | /// 23 | /// indented 24 | /// deindented 25 | /// normal 26 | /// "; 27 | /// 28 | /// let mut lines = split_lines_without_indent(string_literal); 29 | /// assert_eq!(lines.next(), Some("hello world")); 30 | /// assert_eq!(lines.next(), Some("")); 31 | /// assert_eq!(lines.next(), Some(" * bullet 1")); 32 | /// assert_eq!(lines.next(), Some(" * bullet 2")); 33 | /// assert_eq!(lines.next(), Some("")); 34 | /// assert_eq!(lines.next(), Some(" indented")); 35 | /// assert_eq!(lines.next(), Some("deindented")); 36 | /// assert_eq!(lines.next(), Some("normal")); 37 | /// assert_eq!(lines.next(), Some("")); 38 | /// assert_eq!(lines.next(), None); 39 | /// ``` 40 | pub fn split_lines_without_indent(input: &str) -> impl Iterator { 41 | let input = LocatedSpan::new(input); 42 | let (remaining, _) = multispace0::<_, (_, _)>(input).expect("multispace0 cannot fail"); 43 | let (fragment, indent_level) = (remaining.fragment(), remaining.get_column()); 44 | 45 | fragment.split('\n').enumerate().map(move |(i, row)| { 46 | if i > 0 { 47 | let trim_start = row 48 | .char_indices() 49 | .take_while(|(i, c)| c.is_whitespace() && *i < indent_level - 1) 50 | .count(); 51 | &row[trim_start..] 52 | } else { 53 | row 54 | } 55 | }) 56 | } 57 | 58 | #[cfg(test)] 59 | mod tests { 60 | use super::*; 61 | 62 | #[test] 63 | fn accepts_empty_strings() { 64 | let lines: Vec<_> = split_lines_without_indent("").collect(); 65 | assert_eq!(lines, vec![""]); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /nix-lexer/src/tokens.rs: -------------------------------------------------------------------------------- 1 | //! Types of tokens and token kinds. 2 | 3 | use std::fmt::{self, Debug, Display, Formatter}; 4 | use std::ops::Range; 5 | 6 | use codespan::Span; 7 | #[cfg(feature = "serialization")] 8 | use serde::{Deserialize, Serialize}; 9 | 10 | /// A token with span information. 11 | #[derive(Clone, Debug, Eq, PartialEq)] 12 | #[cfg_attr(feature = "serialization", derive(Deserialize, Serialize))] 13 | pub struct Token { 14 | /// The kind of token. 15 | pub kind: TokenKind, 16 | /// The span of the token. 17 | pub span: Span, 18 | } 19 | 20 | impl Token { 21 | /// Creates a new `Token` from the given `kind` and `span`. 22 | pub const fn new(kind: TokenKind, span: Span) -> Self { 23 | Token { kind, span } 24 | } 25 | 26 | /// Returns whether the given token is a kind of trivia, i.e. whitespace and comments. 27 | pub fn is_trivia(&self) -> bool { 28 | self.kind.is_trivia() 29 | } 30 | 31 | /// Returns an object which implements `Display` which can be used to display a token, given a 32 | /// slice of source text. 33 | pub const fn display<'s>(&self, source: &'s str) -> DisplayToken<'s> { 34 | DisplayToken { 35 | source, 36 | kind: self.kind, 37 | span: self.span, 38 | } 39 | } 40 | } 41 | 42 | /// Helper struct for printing tokens with `format!` and `{}`. 43 | pub struct DisplayToken<'s> { 44 | source: &'s str, 45 | kind: TokenKind, 46 | span: Span, 47 | } 48 | 49 | impl Debug for DisplayToken<'_> { 50 | fn fmt(&self, f: &mut Formatter) -> fmt::Result { 51 | f.debug_struct(stringify!(Token)) 52 | .field("kind", &self.kind) 53 | .field("span", &self.span) 54 | .finish() 55 | } 56 | } 57 | 58 | impl Display for DisplayToken<'_> { 59 | fn fmt(&self, f: &mut Formatter) -> fmt::Result { 60 | let range = Range::from(self.span); 61 | write!( 62 | f, 63 | "Kind: {:?}, Span: {}, Text: {:?}", 64 | self.kind, self.span, &self.source[range] 65 | ) 66 | } 67 | } 68 | 69 | /// A list specifying all possible tokens. 70 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] 71 | #[cfg_attr(feature = "serialization", derive(Deserialize, Serialize))] 72 | pub enum TokenKind { 73 | /// `# foo` 74 | LineComment, 75 | /// `/* foo */` 76 | BlockComment { terminated: bool }, 77 | /// Spaces, tabs, and newlines 78 | Whitespace, 79 | 80 | /// `foo`, `_`, `let` 81 | Ident, 82 | /// Any literal value 83 | Literal { kind: LiteralKind }, 84 | /// `"`, `''` 85 | StringTerm { kind: StringKind }, 86 | /// Text span enclosed in one or more `StringTerm`s 87 | StringLiteral, 88 | 89 | // Operators 90 | /// `+` 91 | Add, 92 | /// `-` 93 | Sub, 94 | /// `*` 95 | Mul, 96 | /// `/` 97 | Div, 98 | /// `==` 99 | IsEq, 100 | /// `!=` 101 | NotEq, 102 | /// `<` 103 | LessThan, 104 | /// `<=` 105 | LessThanEq, 106 | /// `>` 107 | GreaterThan, 108 | /// `>=` 109 | GreaterThanEq, 110 | /// `&&` 111 | LogicalAnd, 112 | /// `||` 113 | LogicalOr, 114 | /// `++` 115 | Concat, 116 | /// `//` 117 | Update, 118 | /// `?` 119 | Question, 120 | /// `->` 121 | Imply, 122 | /// `!` 123 | Not, 124 | 125 | /// `@` 126 | At, 127 | /// `:` 128 | Colon, 129 | /// `,` 130 | Comma, 131 | /// `.` 132 | Dot, 133 | /// `...` 134 | Ellipsis, 135 | /// `=` 136 | Eq, 137 | /// `${` 138 | Interpolate, 139 | /// `{` 140 | OpenBrace, 141 | /// `}` 142 | CloseBrace, 143 | /// `[` 144 | OpenBracket, 145 | /// `]` 146 | CloseBracket, 147 | /// `(` 148 | OpenParen, 149 | /// `)` 150 | CloseParen, 151 | /// `;` 152 | Semi, 153 | 154 | /// Any token unknown to the lexer, e.g. `^` 155 | Unknown, 156 | } 157 | 158 | impl TokenKind { 159 | /// Returns whether the given token is a kind of trivia, i.e. whitespace and comments. 160 | pub fn is_trivia(self) -> bool { 161 | match self { 162 | TokenKind::LineComment | TokenKind::BlockComment { .. } | TokenKind::Whitespace => true, 163 | _ => false, 164 | } 165 | } 166 | 167 | /// Returns a static description of a `TokenKind` suitable for error reporting. 168 | pub fn description(self) -> &'static str { 169 | match self { 170 | TokenKind::LineComment => "line comment", 171 | TokenKind::BlockComment { .. } => "block comment", 172 | TokenKind::Whitespace => "whitespace", 173 | 174 | TokenKind::Ident => "identifier", 175 | TokenKind::Literal { kind } => match kind { 176 | LiteralKind::Float => "float literal", 177 | LiteralKind::Integer => "integer literal", 178 | LiteralKind::Path { .. } => "path literal", 179 | LiteralKind::PathTemplate { .. } => "path template literal", 180 | LiteralKind::Uri => "URI literal", 181 | }, 182 | TokenKind::StringTerm { kind } => match kind { 183 | StringKind::Normal => "`\"`", 184 | StringKind::Indented => "`''`", 185 | }, 186 | TokenKind::StringLiteral => "string literal", 187 | 188 | TokenKind::Add => "`+`", 189 | TokenKind::Sub => "`-`", 190 | TokenKind::Mul => "`*`", 191 | TokenKind::Div => "`/`", 192 | 193 | TokenKind::IsEq => "`==`", 194 | TokenKind::NotEq => "`!=`", 195 | TokenKind::LessThan => "`<`", 196 | TokenKind::LessThanEq => "`<=`", 197 | TokenKind::GreaterThan => "`>`", 198 | TokenKind::GreaterThanEq => "`>=`", 199 | TokenKind::LogicalAnd => "`&&`", 200 | TokenKind::LogicalOr => "`||`", 201 | TokenKind::Concat => "`&&`", 202 | TokenKind::Update => "`//`", 203 | TokenKind::Question => "`?`", 204 | TokenKind::Imply => "`->`", 205 | TokenKind::Not => "`!`", 206 | 207 | TokenKind::At => "`@`", 208 | TokenKind::Colon => "`:`", 209 | TokenKind::Comma => "`,`", 210 | TokenKind::Dot => "`.`", 211 | TokenKind::Ellipsis => "`...`", 212 | TokenKind::Eq => "`=`", 213 | TokenKind::Interpolate => "`${`", 214 | TokenKind::OpenBrace => "`{`", 215 | TokenKind::CloseBrace => "`}`", 216 | TokenKind::OpenBracket => "`[`", 217 | TokenKind::CloseBracket => "`]`", 218 | TokenKind::OpenParen => "`(`", 219 | TokenKind::CloseParen => "`)`", 220 | TokenKind::Semi => "`;`", 221 | 222 | TokenKind::Unknown => "", 223 | } 224 | } 225 | } 226 | 227 | /// A list of valid literal values. 228 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] 229 | #[cfg_attr(feature = "serialization", derive(Deserialize, Serialize))] 230 | pub enum LiteralKind { 231 | /// `3.14` 232 | Float, 233 | /// `1234`, `00001` 234 | Integer, 235 | /// `./foo/bar`, `~/foo/bar`, `/foo/bar`, `foo/bar` 236 | Path { trailing_slash: bool }, 237 | /// ``, `` 238 | PathTemplate { trailing_slash: bool }, 239 | /// `https://github.com/NixOS/nix` 240 | Uri, 241 | } 242 | 243 | /// A list of valid string terminators. 244 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] 245 | #[cfg_attr(feature = "serialization", derive(Deserialize, Serialize))] 246 | pub enum StringKind { 247 | /// A string terminated by double quotes, `"`. 248 | Normal, 249 | /// A string terminated by two single quotes, `''`. 250 | Indented, 251 | } 252 | -------------------------------------------------------------------------------- /nix-lexer/src/unescape.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for validating string literals and turning them into the values that they represent. 2 | 3 | use std::borrow::Cow; 4 | 5 | use super::tokens::StringKind; 6 | 7 | /// Takes the contents of a string literal (without quotes) and returns an unescaped string. 8 | /// 9 | /// # Examples 10 | /// 11 | /// Normal strings: 12 | /// 13 | /// ``` 14 | /// use nix_lexer::{unescape_str, StringKind}; 15 | /// 16 | /// let unescaped = unescape_str("foo \\${bar} \\n\\r\\t \\' \\^ baz", StringKind::Normal); 17 | /// assert_eq!(unescaped, "foo ${bar} \n\r\t ' ^ baz"); 18 | /// ``` 19 | /// 20 | /// Indented strings: 21 | /// 22 | /// ``` 23 | /// use nix_lexer::{unescape_str, StringKind}; 24 | /// 25 | /// let unescaped = unescape_str("foo ''${bar} ''' ''^ baz", StringKind::Indented); 26 | /// assert_eq!(unescaped, "foo ${bar} ' ''^ baz"); 27 | /// ``` 28 | pub fn unescape_str(s: &str, kind: StringKind) -> Cow { 29 | match kind { 30 | StringKind::Normal => unescape_normal_str(s), 31 | StringKind::Indented => unescape_indented_str(s), 32 | } 33 | } 34 | 35 | fn unescape_normal_str(s: &str) -> Cow { 36 | if s.contains('\\') { 37 | let mut res = String::with_capacity(s.len()); 38 | let mut chars = s.chars(); 39 | 40 | while let Some(c) = chars.next() { 41 | if c == '\\' { 42 | match chars.next() { 43 | Some('n') => res.push('\n'), 44 | Some('r') => res.push('\r'), 45 | Some('t') => res.push('\t'), 46 | Some(c) => res.push(c), 47 | None => (), 48 | } 49 | } else { 50 | res.push(c); 51 | } 52 | } 53 | 54 | Cow::Owned(res) 55 | } else { 56 | Cow::Borrowed(s) 57 | } 58 | } 59 | 60 | fn unescape_indented_str(s: &str) -> Cow { 61 | if s.contains("''") { 62 | let mut res = String::with_capacity(s.len()); 63 | let mut chars = s.chars().peekable(); 64 | 65 | while let Some(c) = chars.next() { 66 | if c == '\'' && chars.peek() == Some(&'\'') { 67 | match chars.by_ref().nth(1) { 68 | Some('$') => res.push('$'), 69 | Some('\'') => res.push('\''), 70 | Some(c) => { 71 | res.push('\''); 72 | res.push('\''); 73 | res.push(c); 74 | } 75 | None => res.push('\''), 76 | } 77 | } else { 78 | res.push(c); 79 | } 80 | } 81 | 82 | Cow::Owned(res) 83 | } else { 84 | Cow::Borrowed(s) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /nix-lexer/tests/literals.nix: -------------------------------------------------------------------------------- 1 | _ 2 | _leading_underscore 3 | FooBar123 4 | fooBar123 5 | foo_bar_123 6 | foo-bar-123 7 | let 8 | in 9 | assert 10 | 11 | 123 12 | 00001 13 | .0 14 | .0e4 15 | .0E-4 16 | .0E+4 17 | 123. 18 | 123.45 19 | 123.e45 20 | 123.E45 21 | 123.E+45 22 | 123.E-45 23 | 123.45e67 24 | 25 | ./. 26 | ./foo 27 | ./foo/bar_baz-quux-123 28 | /lost+found 29 | /. 30 | ~/. 31 | ~/foo/bar 32 | /intentionally/invalid/ 33 | 34 | 35 | 36 | 37 | 38 | https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz 39 | 40 | "" 41 | " foo ${bar baz} \" \${quux} " 42 | '' foo ${bar baz} ''' ''${quux} '' 43 | -------------------------------------------------------------------------------- /nix-lexer/tests/literals.snap: -------------------------------------------------------------------------------- 1 | Kind: Ident, Span: [0, 1), Text: "_" 2 | Kind: Whitespace, Span: [1, 2), Text: "\n" 3 | Kind: Ident, Span: [2, 21), Text: "_leading_underscore" 4 | Kind: Whitespace, Span: [21, 22), Text: "\n" 5 | Kind: Ident, Span: [22, 31), Text: "FooBar123" 6 | Kind: Whitespace, Span: [31, 32), Text: "\n" 7 | Kind: Ident, Span: [32, 41), Text: "fooBar123" 8 | Kind: Whitespace, Span: [41, 42), Text: "\n" 9 | Kind: Ident, Span: [42, 53), Text: "foo_bar_123" 10 | Kind: Whitespace, Span: [53, 54), Text: "\n" 11 | Kind: Ident, Span: [54, 65), Text: "foo-bar-123" 12 | Kind: Whitespace, Span: [65, 66), Text: "\n" 13 | Kind: Ident, Span: [66, 69), Text: "let" 14 | Kind: Whitespace, Span: [69, 70), Text: "\n" 15 | Kind: Ident, Span: [70, 72), Text: "in" 16 | Kind: Whitespace, Span: [72, 73), Text: "\n" 17 | Kind: Ident, Span: [73, 79), Text: "assert" 18 | Kind: Whitespace, Span: [79, 81), Text: "\n\n" 19 | Kind: Literal { kind: Integer }, Span: [81, 84), Text: "123" 20 | Kind: Whitespace, Span: [84, 85), Text: "\n" 21 | Kind: Literal { kind: Integer }, Span: [85, 90), Text: "00001" 22 | Kind: Whitespace, Span: [90, 91), Text: "\n" 23 | Kind: Literal { kind: Float }, Span: [91, 93), Text: ".0" 24 | Kind: Whitespace, Span: [93, 94), Text: "\n" 25 | Kind: Literal { kind: Float }, Span: [94, 98), Text: ".0e4" 26 | Kind: Whitespace, Span: [98, 99), Text: "\n" 27 | Kind: Literal { kind: Float }, Span: [99, 104), Text: ".0E-4" 28 | Kind: Whitespace, Span: [104, 105), Text: "\n" 29 | Kind: Literal { kind: Float }, Span: [105, 110), Text: ".0E+4" 30 | Kind: Whitespace, Span: [110, 111), Text: "\n" 31 | Kind: Literal { kind: Float }, Span: [111, 115), Text: "123." 32 | Kind: Whitespace, Span: [115, 116), Text: "\n" 33 | Kind: Literal { kind: Float }, Span: [116, 122), Text: "123.45" 34 | Kind: Whitespace, Span: [122, 123), Text: "\n" 35 | Kind: Literal { kind: Float }, Span: [123, 130), Text: "123.e45" 36 | Kind: Whitespace, Span: [130, 131), Text: "\n" 37 | Kind: Literal { kind: Float }, Span: [131, 138), Text: "123.E45" 38 | Kind: Whitespace, Span: [138, 139), Text: "\n" 39 | Kind: Literal { kind: Float }, Span: [139, 147), Text: "123.E+45" 40 | Kind: Whitespace, Span: [147, 148), Text: "\n" 41 | Kind: Literal { kind: Float }, Span: [148, 156), Text: "123.E-45" 42 | Kind: Whitespace, Span: [156, 157), Text: "\n" 43 | Kind: Literal { kind: Float }, Span: [157, 166), Text: "123.45e67" 44 | Kind: Whitespace, Span: [166, 168), Text: "\n\n" 45 | Kind: Literal { kind: Path { trailing_slash: false } }, Span: [168, 171), Text: "./." 46 | Kind: Whitespace, Span: [171, 172), Text: "\n" 47 | Kind: Literal { kind: Path { trailing_slash: false } }, Span: [172, 177), Text: "./foo" 48 | Kind: Whitespace, Span: [177, 178), Text: "\n" 49 | Kind: Literal { kind: Path { trailing_slash: false } }, Span: [178, 200), Text: "./foo/bar_baz-quux-123" 50 | Kind: Whitespace, Span: [200, 201), Text: "\n" 51 | Kind: Literal { kind: Path { trailing_slash: false } }, Span: [201, 212), Text: "/lost+found" 52 | Kind: Whitespace, Span: [212, 213), Text: "\n" 53 | Kind: Literal { kind: Path { trailing_slash: false } }, Span: [213, 215), Text: "/." 54 | Kind: Whitespace, Span: [215, 216), Text: "\n" 55 | Kind: Literal { kind: Path { trailing_slash: false } }, Span: [216, 219), Text: "~/." 56 | Kind: Whitespace, Span: [219, 220), Text: "\n" 57 | Kind: Literal { kind: Path { trailing_slash: false } }, Span: [220, 229), Text: "~/foo/bar" 58 | Kind: Whitespace, Span: [229, 230), Text: "\n" 59 | Kind: Literal { kind: Path { trailing_slash: true } }, Span: [230, 253), Text: "/intentionally/invalid/" 60 | Kind: Whitespace, Span: [253, 255), Text: "\n\n" 61 | Kind: Literal { kind: PathTemplate { trailing_slash: false } }, Span: [255, 264), Text: "" 62 | Kind: Whitespace, Span: [264, 265), Text: "\n" 63 | Kind: Literal { kind: PathTemplate { trailing_slash: false } }, Span: [265, 291), Text: "" 64 | Kind: Whitespace, Span: [291, 292), Text: "\n" 65 | Kind: Literal { kind: PathTemplate { trailing_slash: true } }, Span: [292, 316), Text: "" 66 | Kind: Whitespace, Span: [316, 318), Text: "\n\n" 67 | Kind: Literal { kind: Uri }, Span: [318, 382), Text: "https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz" 68 | Kind: Whitespace, Span: [382, 384), Text: "\n\n" 69 | Kind: StringTerm { kind: Normal }, Span: [384, 385), Text: "\"" 70 | Kind: StringTerm { kind: Normal }, Span: [385, 386), Text: "\"" 71 | Kind: Whitespace, Span: [386, 387), Text: "\n" 72 | Kind: StringTerm { kind: Normal }, Span: [387, 388), Text: "\"" 73 | Kind: StringLiteral, Span: [388, 393), Text: " foo " 74 | Kind: Interpolate, Span: [393, 395), Text: "${" 75 | Kind: Ident, Span: [395, 398), Text: "bar" 76 | Kind: Whitespace, Span: [398, 399), Text: " " 77 | Kind: Ident, Span: [399, 402), Text: "baz" 78 | Kind: CloseBrace, Span: [402, 403), Text: "}" 79 | Kind: StringLiteral, Span: [403, 416), Text: " \\\" \\${quux} " 80 | Kind: StringTerm { kind: Normal }, Span: [416, 417), Text: "\"" 81 | Kind: Whitespace, Span: [417, 418), Text: "\n" 82 | Kind: StringTerm { kind: Indented }, Span: [418, 420), Text: "\'\'" 83 | Kind: StringLiteral, Span: [420, 425), Text: " foo " 84 | Kind: Interpolate, Span: [425, 427), Text: "${" 85 | Kind: Ident, Span: [427, 430), Text: "bar" 86 | Kind: Whitespace, Span: [430, 431), Text: " " 87 | Kind: Ident, Span: [431, 434), Text: "baz" 88 | Kind: CloseBrace, Span: [434, 435), Text: "}" 89 | Kind: StringLiteral, Span: [435, 450), Text: " \'\'\' \'\'${quux} " 90 | Kind: StringTerm { kind: Indented }, Span: [450, 452), Text: "\'\'" 91 | Kind: Whitespace, Span: [452, 453), Text: "\n" -------------------------------------------------------------------------------- /nix-lexer/tests/operators.nix: -------------------------------------------------------------------------------- 1 | + - * / == != < <= > >= && || ++ // ? -> ! 2 | -------------------------------------------------------------------------------- /nix-lexer/tests/operators.snap: -------------------------------------------------------------------------------- 1 | Kind: Add, Span: [0, 1), Text: "+" 2 | Kind: Whitespace, Span: [1, 2), Text: " " 3 | Kind: Sub, Span: [2, 3), Text: "-" 4 | Kind: Whitespace, Span: [3, 4), Text: " " 5 | Kind: Mul, Span: [4, 5), Text: "*" 6 | Kind: Whitespace, Span: [5, 6), Text: " " 7 | Kind: Div, Span: [6, 7), Text: "/" 8 | Kind: Whitespace, Span: [7, 8), Text: " " 9 | Kind: IsEq, Span: [8, 10), Text: "==" 10 | Kind: Whitespace, Span: [10, 11), Text: " " 11 | Kind: NotEq, Span: [11, 13), Text: "!=" 12 | Kind: Whitespace, Span: [13, 14), Text: " " 13 | Kind: LessThan, Span: [14, 15), Text: "<" 14 | Kind: Whitespace, Span: [15, 16), Text: " " 15 | Kind: LessThanEq, Span: [16, 18), Text: "<=" 16 | Kind: Whitespace, Span: [18, 19), Text: " " 17 | Kind: GreaterThan, Span: [19, 20), Text: ">" 18 | Kind: Whitespace, Span: [20, 21), Text: " " 19 | Kind: GreaterThanEq, Span: [21, 23), Text: ">=" 20 | Kind: Whitespace, Span: [23, 24), Text: " " 21 | Kind: LogicalAnd, Span: [24, 26), Text: "&&" 22 | Kind: Whitespace, Span: [26, 27), Text: " " 23 | Kind: LogicalOr, Span: [27, 29), Text: "||" 24 | Kind: Whitespace, Span: [29, 30), Text: " " 25 | Kind: Concat, Span: [30, 32), Text: "++" 26 | Kind: Whitespace, Span: [32, 33), Text: " " 27 | Kind: Update, Span: [33, 35), Text: "//" 28 | Kind: Whitespace, Span: [35, 36), Text: " " 29 | Kind: Question, Span: [36, 37), Text: "?" 30 | Kind: Whitespace, Span: [37, 38), Text: " " 31 | Kind: Imply, Span: [38, 40), Text: "->" 32 | Kind: Whitespace, Span: [40, 41), Text: " " 33 | Kind: Not, Span: [41, 42), Text: "!" 34 | Kind: Whitespace, Span: [42, 43), Text: "\n" -------------------------------------------------------------------------------- /nix-lexer/tests/punctuation.nix: -------------------------------------------------------------------------------- 1 | @ 2 | : 3 | , 4 | . 5 | ... 6 | = 7 | ${ 8 | { 9 | } 10 | [ 11 | ] 12 | ( 13 | ) 14 | ; 15 | ^ 16 | -------------------------------------------------------------------------------- /nix-lexer/tests/punctuation.snap: -------------------------------------------------------------------------------- 1 | Kind: At, Span: [0, 1), Text: "@" 2 | Kind: Whitespace, Span: [1, 2), Text: "\n" 3 | Kind: Colon, Span: [2, 3), Text: ":" 4 | Kind: Whitespace, Span: [3, 4), Text: "\n" 5 | Kind: Comma, Span: [4, 5), Text: "," 6 | Kind: Whitespace, Span: [5, 6), Text: "\n" 7 | Kind: Dot, Span: [6, 7), Text: "." 8 | Kind: Whitespace, Span: [7, 8), Text: "\n" 9 | Kind: Ellipsis, Span: [8, 11), Text: "..." 10 | Kind: Whitespace, Span: [11, 12), Text: "\n" 11 | Kind: Eq, Span: [12, 13), Text: "=" 12 | Kind: Whitespace, Span: [13, 14), Text: "\n" 13 | Kind: Interpolate, Span: [14, 16), Text: "${" 14 | Kind: Whitespace, Span: [16, 17), Text: "\n" 15 | Kind: OpenBrace, Span: [17, 18), Text: "{" 16 | Kind: Whitespace, Span: [18, 19), Text: "\n" 17 | Kind: CloseBrace, Span: [19, 20), Text: "}" 18 | Kind: Whitespace, Span: [20, 21), Text: "\n" 19 | Kind: OpenBracket, Span: [21, 22), Text: "[" 20 | Kind: Whitespace, Span: [22, 23), Text: "\n" 21 | Kind: CloseBracket, Span: [23, 24), Text: "]" 22 | Kind: Whitespace, Span: [24, 25), Text: "\n" 23 | Kind: OpenParen, Span: [25, 26), Text: "(" 24 | Kind: Whitespace, Span: [26, 27), Text: "\n" 25 | Kind: CloseParen, Span: [27, 28), Text: ")" 26 | Kind: Whitespace, Span: [28, 29), Text: "\n" 27 | Kind: Semi, Span: [29, 30), Text: ";" 28 | Kind: Whitespace, Span: [30, 31), Text: "\n" 29 | Kind: Unknown, Span: [31, 32), Text: "^" 30 | Kind: Whitespace, Span: [32, 33), Text: "\n" -------------------------------------------------------------------------------- /nix-lexer/tests/snapshots.rs: -------------------------------------------------------------------------------- 1 | use codespan::Files; 2 | 3 | macro_rules! assert_tokens_match { 4 | ($expression_file_name:ident) => { 5 | #[test] 6 | fn $expression_file_name() { 7 | let mut files = Files::new(); 8 | let file_id = files.add( 9 | stringify!($expression_file_name), 10 | include_str!(concat!( 11 | env!("CARGO_MANIFEST_DIR"), 12 | "/tests/", 13 | stringify!($expression_file_name), 14 | ".nix" 15 | )), 16 | ); 17 | 18 | let source = files.source(file_id); 19 | let actual = nix_lexer::tokenize(&source).map(|t| t.display(&source).to_string()); 20 | 21 | let expected = include_str!(concat!( 22 | env!("CARGO_MANIFEST_DIR"), 23 | "/tests/", 24 | stringify!($expression_file_name), 25 | ".snap" 26 | )) 27 | .split_terminator('\n'); 28 | 29 | for (actual, expect) in actual.zip(expected) { 30 | assert_eq!(actual, expect); 31 | } 32 | } 33 | }; 34 | 35 | (update $expression_file_name:ident) => { 36 | #[test] 37 | fn $expression_file_name() { 38 | let mut files = Files::new(); 39 | let file_id = files.add( 40 | stringify!($expression_file_name), 41 | include_str!(concat!( 42 | env!("CARGO_MANIFEST_DIR"), 43 | "/tests/", 44 | stringify!($expression_file_name), 45 | ".nix" 46 | )), 47 | ); 48 | 49 | let source = files.source(file_id); 50 | let tokens: Vec<_> = nix_lexer::tokenize(&source) 51 | .map(|t| t.display(&source).to_string()) 52 | .collect(); 53 | 54 | std::fs::write( 55 | concat!( 56 | env!("CARGO_MANIFEST_DIR"), 57 | "/tests/", 58 | stringify!($expression_file_name), 59 | ".snap" 60 | ), 61 | tokens.join("\n"), 62 | ) 63 | .unwrap(); 64 | } 65 | }; 66 | } 67 | 68 | assert_tokens_match!(trivia); 69 | assert_tokens_match!(literals); 70 | assert_tokens_match!(punctuation); 71 | assert_tokens_match!(operators); 72 | -------------------------------------------------------------------------------- /nix-lexer/tests/trivia.nix: -------------------------------------------------------------------------------- 1 | # foo 2 | # 3 | # bar 4 | 5 | # baz 6 | /* 7 | * 8 | * hello 9 | * 10 | */ 11 | 12 | /* /* foo */ 13 | -------------------------------------------------------------------------------- /nix-lexer/tests/trivia.snap: -------------------------------------------------------------------------------- 1 | Kind: LineComment, Span: [0, 5), Text: "# foo" 2 | Kind: Whitespace, Span: [5, 6), Text: "\n" 3 | Kind: LineComment, Span: [6, 7), Text: "#" 4 | Kind: Whitespace, Span: [7, 8), Text: "\n" 5 | Kind: LineComment, Span: [8, 13), Text: "# bar" 6 | Kind: Whitespace, Span: [13, 16), Text: "\n\n\t" 7 | Kind: LineComment, Span: [16, 21), Text: "# baz" 8 | Kind: Whitespace, Span: [21, 22), Text: "\n" 9 | Kind: BlockComment { terminated: true }, Span: [22, 41), Text: "/*\n *\n * hello\n *\n " 10 | Kind: Mul, Span: [41, 42), Text: "*" 11 | Kind: Div, Span: [42, 43), Text: "/" 12 | Kind: Whitespace, Span: [43, 45), Text: "\n\n" 13 | Kind: BlockComment { terminated: true }, Span: [45, 55), Text: "/* /* foo " 14 | Kind: Mul, Span: [55, 56), Text: "*" 15 | Kind: Div, Span: [56, 57), Text: "/" 16 | Kind: Whitespace, Span: [57, 58), Text: "\n" -------------------------------------------------------------------------------- /nix-parser/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nix-parser" 3 | version = "0.1.0" 4 | authors = ["Eyal Kalderon "] 5 | license = "MIT OR Apache-2.0" 6 | edition = "2018" 7 | 8 | [dependencies] 9 | codespan = "0.8.0" 10 | codespan-reporting = "0.8.0" 11 | lexical-core = "0.6.2" 12 | nom_locate = "1.0.0" 13 | once_cell = "1.1.0" 14 | smallvec = "0.6.12" 15 | url = "2.1.0" 16 | 17 | [dependencies.nom] 18 | version = "5.0.1" 19 | default-features = false 20 | features = ["std"] 21 | 22 | [dependencies.regex] 23 | version = "1.3.1" 24 | default-features = false 25 | features = ["std", "perf"] 26 | 27 | [dev-dependencies] 28 | criterion = "0.3.0" 29 | structopt = "0.3.3" 30 | 31 | [[bench]] 32 | name = "example" 33 | harness = false 34 | -------------------------------------------------------------------------------- /nix-parser/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /nix-parser/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Eyal Kalderon 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /nix-parser/benches/example.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; 4 | use nix_parser::ast::SourceFile; 5 | 6 | const EXAMPLE_FILE: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/example.nix")); 7 | 8 | fn parse_example(b: &mut Criterion) { 9 | let mut group = b.benchmark_group("parse_example"); 10 | 11 | let module = black_box(EXAMPLE_FILE); 12 | group.throughput(Throughput::Bytes(EXAMPLE_FILE.len() as u64)); 13 | group.bench_function("parse example.nix", move |b| { 14 | b.iter_with_large_drop(|| { 15 | SourceFile::from_str(module).expect("Failed to parse example.nix") 16 | }); 17 | }); 18 | 19 | group.finish() 20 | } 21 | 22 | criterion_group!(benches, parse_example); 23 | criterion_main!(benches); 24 | -------------------------------------------------------------------------------- /nix-parser/example.nix: -------------------------------------------------------------------------------- 1 | # Source file comment 2 | 3 | with import {}; 4 | let 5 | # Comment on binding 6 | # 7 | # Header 8 | # 9 | # 1. First 10 | # 2. Second 11 | set = { inherit (foo) test; foo.bar.baz = 12345; outer = { inner = true; }; }; 12 | 13 | # Script for testing string interpolation 14 | value = "here\nare\rsome\trandom\tescape\"codes\$"; 15 | script = pkgs.writeShellScriptBin "interpolationTest" '' 16 | #!/bin/bash 17 | echo '${value}' 18 | ''; 19 | in 20 | stdenv.mkDerivation { 21 | name = "interpolation-test"; 22 | buildInputs = [ script ]; 23 | } 24 | -------------------------------------------------------------------------------- /nix-parser/examples/viewer.rs: -------------------------------------------------------------------------------- 1 | use std::io::Read; 2 | use std::time::Instant; 3 | use std::{fs, io, process}; 4 | 5 | use codespan::Files; 6 | use codespan_reporting::term::termcolor::{ColorChoice, StandardStream}; 7 | use codespan_reporting::term::{emit, Config}; 8 | use nix_parser::ast::SourceFile; 9 | use nix_parser::error::Errors; 10 | use nix_parser::parser::parse_source_file_partial; 11 | use structopt::StructOpt; 12 | 13 | #[derive(StructOpt)] 14 | #[structopt(about = "Parses a Nix expression and displays the result")] 15 | struct Cli { 16 | /// Display the abstract syntax tree (AST) 17 | #[structopt(long)] 18 | debug: bool, 19 | /// Continue even in the presence of parse errors 20 | #[structopt(short, long)] 21 | partial: bool, 22 | /// Produce pretty printed output 23 | #[structopt(long)] 24 | pretty: bool, 25 | /// Display the amount of time spent parsing 26 | #[structopt(long)] 27 | time: bool, 28 | /// Source file to parse (reads from stdin if "-") 29 | #[structopt(default_value = "default.nix")] 30 | file: String, 31 | } 32 | 33 | fn main() { 34 | let args = Cli::from_args(); 35 | 36 | let expr = if args.file == "-" { 37 | let mut expr = String::new(); 38 | io::stdin() 39 | .read_to_string(&mut expr) 40 | .map(|_| expr) 41 | .expect("Failed to read expression from stdin") 42 | } else { 43 | fs::read_to_string(&args.file).expect("Unable to read file") 44 | }; 45 | 46 | if args.partial { 47 | parse_partial(&expr, args); 48 | } else { 49 | parse_full(&expr, args); 50 | } 51 | } 52 | 53 | fn parse_full(expr: &str, args: Cli) { 54 | let start = Instant::now(); 55 | let ast: SourceFile = expr.parse().unwrap_or_else(|errors| { 56 | print_diagnostics(expr, &errors); 57 | process::exit(1); 58 | }); 59 | let end = start.elapsed(); 60 | 61 | print_source_file(&ast, &args); 62 | 63 | if args.time { 64 | println!("\nParse time: {:?}", end); 65 | } 66 | } 67 | 68 | fn parse_partial(expr: &str, args: Cli) { 69 | let start = Instant::now(); 70 | let partial = parse_source_file_partial(expr).unwrap_or_else(|errors| { 71 | print_diagnostics(expr, &errors); 72 | process::exit(1); 73 | }); 74 | let end = start.elapsed(); 75 | 76 | if let Some(ref ast) = partial.value() { 77 | print_source_file(ast, &args) 78 | } else { 79 | println!("No expression value produced"); 80 | } 81 | 82 | if !partial.errors().is_empty() { 83 | print!("\n"); 84 | print_diagnostics(expr, partial.errors()); 85 | } 86 | 87 | if args.time { 88 | println!("\nParse time: {:?}", end); 89 | } 90 | } 91 | 92 | fn print_source_file(ast: &SourceFile, args: &Cli) { 93 | if args.debug { 94 | if args.pretty { 95 | println!("{:#?}", ast); 96 | } else { 97 | println!("{:?}", ast); 98 | } 99 | } else { 100 | if args.pretty { 101 | println!("{:#}", ast); 102 | } else { 103 | println!("{}", ast); 104 | } 105 | } 106 | } 107 | 108 | fn print_diagnostics(expr: &str, errors: &Errors) { 109 | let mut files = Files::new(); 110 | let id = files.add("stdin", expr); 111 | let diagnostics = errors.to_diagnostics(id); 112 | 113 | let mut lock = StandardStream::stdout(ColorChoice::Auto); 114 | let config = Config::default(); 115 | 116 | for diag in diagnostics { 117 | emit(&mut lock, &config, &files, &diag).unwrap(); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /nix-parser/grammar.abnf: -------------------------------------------------------------------------------- 1 | ; ABNF (RFC5234) syntax for Nix 2 | ; Use UTF-8 encoding 3 | ; 4 | end-of-line = %x0A / %x0D.0A 5 | ellipsis = %x2E.2E.2E ; ... 6 | comment-line = %x23 chars end-of-line 7 | single-quote = %x27 ; ' 8 | double-quote = %x22 ; " 9 | backslash = %x5C ; \ 10 | dollar = %x24 ; $ 11 | left-curly-bracket = %x7B ; { 12 | right-curly-bracket = %x7D ; } 13 | left-square-bracket = %x5B ; [ 14 | right-square-bracket = %x5D ; ] 15 | left-round-bracket = %x28 ; ( 16 | right-round-bracket = %x29 ; ) 17 | at = %x40 ; @ 18 | let = %x6C.65.74 ; let 19 | in = %x69.6E ; in 20 | question = %x3F ; ? 21 | if = %x69.66 ; if 22 | then = %x74.68.65.6E ; then 23 | else = %x65.6C.73.65 ; else 24 | semicolon = %x3B ; ; 25 | colon = %x3A ; : 26 | comma = %x2C ; , 27 | with = %x77.69.74.68; with 28 | assert = %x61.73.73.65.72.74 ; assert 29 | or = %x6F.72 ; or 30 | rec = %x72.65.63 ; rec 31 | inherit = %x69.6E.68.65.72.69.74 ; inherit 32 | plus = %x2B; + 33 | minus = %x2D ; - 34 | hyphen = minus ; - 35 | multiply = %x2A ; * 36 | divide = %x2F ; / 37 | dot = %x2E ; . 38 | not = %x21 ; ! 39 | less = %x3C ; < 40 | less-equal = %x3C.3D ; <= 41 | equal = %x3D ; = 42 | greater = %x3E ; > 43 | greater-equal = %x3E.3D ; >= 44 | set-update = %x2F.2F ; // 45 | logic-imply = %x2D.3E ; -> 46 | logic-equal = %x3D.3D ; == 47 | logic-not-equal = %x21.3D ; != 48 | logic-and = %x26.26 ; && 49 | logic-or = %x7C.7C ; || 50 | antiquote-start = %x24.7B ; ${ 51 | antiquote-end = right-curly-bracket ; } 52 | 53 | double-quote-string = double-quote double-quote-string-continue ; no backtracking on second non-terminal 54 | 55 | double-quote-escapable = backslash / double-quote / dollar / "n" / "t" / "r" 56 | 57 | ; tokenizer: there is no token separator for this non-terminal and preserve new lines and tabs 58 | ; folding non-terminal 59 | double-quote-string-continue = 60 | double-quote 61 | / backslash double-quote-escapable double-quote-string-continue ; no backtracking on second non-terminal 62 | / dollar left-curly-bracket expr right-curly-bracket double-quote-string-continue ; no backtracking on second non-terminal 63 | 64 | single-quote-string = 2*2(single-quote) single-quote-string-continue ; no backtracking on second non-terminal 65 | 66 | ; tokenizer: there is no token separator for this non-terminal and preserve new lines and tabs 67 | ; folding non-terminal 68 | single-quote-string-continue = 69 | 3*3(single-quote) single-quote-string-continue ; backtrack 70 | / 2*2(single-quote) dollar single-quote-string-continue ; backtrack 71 | / 2*2(single-quote) backslash single-quote-string-continue ; backtrack 72 | / dollar left-curly-bracket expr right-curly-bracket single-quote-string-continue ; no backtracking on third non-terminal 73 | / 2*2(single-quote) ; backtrack, terminated 74 | / anychar single-quote-string-continue 75 | 76 | expr = 77 | with expr semicolon ; backtrack 78 | / assert expr semicolon ; backtrack 79 | / let set-bindings in expr ; backtrack 80 | / fn-decl 81 | 82 | ; expression with operators at lowest precedences 83 | / logic-imply-expr 84 | / sum-expr 85 | 86 | ; function 87 | ; the following non-terminal is an alias in the sense that 88 | ; it should be "inlined" in all references to it, and thus 89 | ; backtracking is "inlined" in the parsing logic of the 90 | ; upper level non-terminals, instead of producing fatal 91 | ; parser error 92 | fn-decl = 93 | left-curly-bracket formal-args right-curly-bracket [ at ident ] colon expr ; backtrack 94 | / ident at left-curly-bracket formal-args right-curly-bracket colon expr ; backtrack 95 | / ident colon expr ; backtrack to upper level non-terminals 96 | 97 | formal-args = 98 | ellipsis 99 | / ident [ question expr ] comma [ formal-args ] 100 | 101 | logic-imply-expr = logic-expr logic-imply (logic-imply-expr / logic-expr) 102 | 103 | logic-or-expr = logic-and-expr [ logic-or logic-or-expr ] 104 | 105 | logic-and-expr = cmp-expr [ logic-and logic-and-expr ] 106 | 107 | cmp-expr = sum-expr [ ( less / greater / less-equal / greater-equal / logic-equal / logic-not-equal ) sum-expr ] 108 | 109 | ; left-associative non-terminal 110 | sum-expr = prod-expr plus sum-expr / prod-expr minus sum-expr / prod-expr 111 | 112 | ; left-associative non-terminal 113 | prod-expr = unary-expr multiply prod-expr / unary-expr divide prod-expr / unary-expr 114 | 115 | unary-expr = not membership-expr / minus membership-expr / membership-expr 116 | 117 | ; try non-terminal 118 | membership-expr = 119 | app-expr [ 120 | question ident 121 | / question antiquote-start expr antiquote-end 122 | / question string 123 | ] 124 | 125 | unary-expr-in-list = not membership-expr-in-list / minus membership-expr-in-list / membership-expr-in-list 126 | 127 | ; try non-terminal 128 | membership-expr-in-list = 129 | atomic-expr [ 130 | question ident 131 | / question antiquote-start expr antiquote-end 132 | / question string 133 | ] 134 | 135 | ; left-associative non-terminal 136 | app-expr = atomic-expr app-expr / atomic-expr 137 | 138 | atomic-expr = 139 | single-quote-string 140 | / double-quote-string 141 | ; set family 142 | / [rec / let] left-curly-bracket set-bindings right-curly-bracket ; no backtracking on second non-terminal, soft-bail on second non-terminal terminating on third non-terminal 143 | 144 | ; sub expr 145 | / left-round-bracket expr right-round-bracket ; no backtracking on second non-terminal, soft-bail on second non-terminal terminating on third non-terminal 146 | 147 | ; list 148 | / left-square-bracket list-elements right-square-bracket ; no backtracking on second non-terminal, soft-bail on second non-terminal terminating on third non-terminal 149 | 150 | ; conditional 151 | / if expr then expr else expr ; no backtracking on second non-terminal, soft-bail on second non-terminal terminating on third non-terminal; soft-bail on fourth non-terminal terminating on fifth non-terminal 152 | 153 | / path 154 | / url 155 | / proj-expr 156 | / ident 157 | / number 158 | 159 | proj-expr = initial-member-access [ or unary-expr ] 160 | 161 | ; left-associative non-terminal 162 | initial-member-access = ident dot member-access 163 | member-access = 164 | string [ dot member-access ] 165 | / antiquote-start expr antiquote-end [ dot member-access ] 166 | / ident expr antiquote-end [ dot member-access ] 167 | 168 | ident-list = (ident / antiquote-start expr antiquote-end) [ ident-list ] 169 | 170 | set-bindings = 171 | inherit [ left-round-bracket expr right-round-bracket ] ident-list semicolon [ set-bindings ] ; no backtracking on third non-terminal, soft-bail on second non-terminal 172 | / antiquote-start expr antiquote-end equal expr semicolon [ set-bindings ] ; no backtracking on third non-terminal, soft-bail on second and fifth non-terminal 173 | / ident equal expr semicolon [ set-bindings ] ; no backtracking on third non-terminal, soft-bail on third non-terminal 174 | 175 | list-elements = unary-expr-in-list [ list-elements ] 176 | 177 | path = PATH-REGEX 178 | url = URL-REGEX 179 | ident = ident-REGEX 180 | number = number-REGEX 181 | -------------------------------------------------------------------------------- /nix-parser/src/ast/fmt.rs: -------------------------------------------------------------------------------- 1 | //! Extension traits and types for `std::fmt`. 2 | 3 | use std::fmt::{Formatter, Result as FmtResult, Write}; 4 | 5 | /// A trait which extends the functionality of [`std::fmt::Formatter`]. 6 | /// 7 | /// [`std::fmt::Formatter`]: https://doc.rust-lang.org/std/fmt/struct.Formatter.html 8 | pub trait FormatterExt<'a> { 9 | /// Indents the given formatter with the given number of spaces. 10 | fn indent<'b>(&'b mut self, level: usize) -> Indented<'b, 'a>; 11 | } 12 | 13 | impl<'a> FormatterExt<'a> for Formatter<'a> { 14 | fn indent<'b>(&'b mut self, level: usize) -> Indented<'b, 'a> { 15 | Indented { 16 | fmt: self, 17 | level, 18 | newline: true, 19 | } 20 | } 21 | } 22 | 23 | /// Formatter which indents each line by a certain amount. 24 | #[allow(missing_debug_implementations)] 25 | pub struct Indented<'a, 'b: 'a> { 26 | fmt: &'a mut Formatter<'b>, 27 | level: usize, 28 | newline: bool, 29 | } 30 | 31 | impl<'a, 'b: 'a> Write for Indented<'a, 'b> { 32 | fn write_str(&mut self, s: &str) -> FmtResult { 33 | for c in s.chars() { 34 | if c == '\n' { 35 | self.fmt.write_char(c)?; 36 | self.newline = true; 37 | continue; 38 | } 39 | 40 | if self.newline { 41 | write!(self.fmt, "{:indent$}", "", indent = self.level)?; 42 | } 43 | 44 | self.fmt.write_char(c)?; 45 | self.newline = false; 46 | } 47 | 48 | Ok(()) 49 | } 50 | } 51 | 52 | #[cfg(test)] 53 | mod tests { 54 | use std::fmt::Display; 55 | 56 | use super::*; 57 | 58 | #[derive(Debug)] 59 | struct Indented(T); 60 | 61 | impl Display for Indented { 62 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 63 | write!(fmt.indent(2), "{}", self.0) 64 | } 65 | } 66 | 67 | #[test] 68 | fn display_with_indentation() { 69 | assert_eq!(" one", Indented("one").to_string()); 70 | assert_eq!(" two", Indented(Indented("two")).to_string()); 71 | assert_eq!(" multi\n line\n", Indented("multi\nline\n").to_string()); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /nix-parser/src/ast/tokens.rs: -------------------------------------------------------------------------------- 1 | //! Nodes representing primitive tokens. 2 | 3 | use std::cmp::Ordering; 4 | use std::fmt::{Display, Formatter, Result as FmtResult}; 5 | use std::path::{Path, PathBuf}; 6 | 7 | use codespan::Span; 8 | use url::Url; 9 | 10 | use crate::{HasSpan, ToSpan}; 11 | 12 | #[derive(Clone, Debug, Eq)] 13 | pub struct Comment(String, Span); 14 | 15 | impl Display for Comment { 16 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 17 | self.0 18 | .lines() 19 | .try_for_each(|line| writeln!(fmt, "#{}", line)) 20 | } 21 | } 22 | 23 | impl<'a> From<&'a str> for Comment { 24 | fn from(s: &'a str) -> Self { 25 | Comment(s.to_owned(), Span::initial()) 26 | } 27 | } 28 | 29 | impl From for Comment { 30 | fn from(s: String) -> Self { 31 | Comment(s, Span::initial()) 32 | } 33 | } 34 | 35 | impl From<(T, S)> for Comment 36 | where 37 | T: Into, 38 | S: ToSpan, 39 | { 40 | fn from((string, span): (T, S)) -> Self { 41 | Comment(string.into(), span.to_span()) 42 | } 43 | } 44 | 45 | impl HasSpan for Comment { 46 | fn span(&self) -> Span { 47 | self.1 48 | } 49 | } 50 | 51 | impl PartialEq for Comment { 52 | fn eq(&self, other: &Self) -> bool { 53 | self.0 == other.0 54 | } 55 | } 56 | 57 | impl PartialOrd for Comment { 58 | fn partial_cmp(&self, other: &Self) -> Option { 59 | self.0.partial_cmp(&other.0) 60 | } 61 | } 62 | 63 | #[derive(Clone, Debug, Eq)] 64 | pub struct Ident(String, Span); 65 | 66 | impl Display for Ident { 67 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 68 | write!(fmt, "{}", self.0) 69 | } 70 | } 71 | 72 | impl<'a> From<&'a str> for Ident { 73 | fn from(s: &'a str) -> Self { 74 | Ident(s.to_owned(), Span::initial()) 75 | } 76 | } 77 | 78 | impl From for Ident { 79 | fn from(s: String) -> Self { 80 | Ident(s, Span::initial()) 81 | } 82 | } 83 | 84 | impl From<(T, S)> for Ident 85 | where 86 | T: Into, 87 | S: ToSpan, 88 | { 89 | fn from((string, span): (T, S)) -> Self { 90 | Ident(string.into(), span.to_span()) 91 | } 92 | } 93 | 94 | impl HasSpan for Ident { 95 | fn span(&self) -> Span { 96 | self.1 97 | } 98 | } 99 | 100 | impl PartialEq for Ident { 101 | fn eq(&self, other: &Self) -> bool { 102 | self.0 == other.0 103 | } 104 | } 105 | 106 | impl PartialOrd for Ident { 107 | fn partial_cmp(&self, other: &Self) -> Option { 108 | self.0.partial_cmp(&other.0) 109 | } 110 | } 111 | 112 | #[derive(Clone, Debug)] 113 | pub enum Literal { 114 | Null(Span), 115 | Boolean(bool, Span), 116 | Float(f64, Span), 117 | Integer(i64, Span), 118 | Path(PathBuf, Span), 119 | PathTemplate(PathBuf, Span), 120 | Uri(Url, Span), 121 | } 122 | 123 | impl Display for Literal { 124 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 125 | match *self { 126 | Literal::Null(_) => write!(fmt, "null"), 127 | Literal::Boolean(ref b, _) => write!(fmt, "{}", b), 128 | Literal::Float(ref f, _) => write!(fmt, "{:?}", f), 129 | Literal::Integer(ref i, _) => write!(fmt, "{}", i), 130 | Literal::Path(ref p, _) => write!(fmt, "{}", p.to_string_lossy()), 131 | Literal::PathTemplate(ref p, _) => write!(fmt, "<{}>", p.to_string_lossy()), 132 | Literal::Uri(ref u, _) => write!(fmt, "{}", u), 133 | } 134 | } 135 | } 136 | 137 | impl> From> for Literal { 138 | fn from(value: Option) -> Self { 139 | value 140 | .map(Into::into) 141 | .unwrap_or_else(|| Literal::Null(Span::initial())) 142 | } 143 | } 144 | 145 | impl From<()> for Literal { 146 | fn from(_: ()) -> Self { 147 | Literal::Null(Span::initial()) 148 | } 149 | } 150 | 151 | impl From for Literal { 152 | fn from(boolean: bool) -> Self { 153 | Literal::Boolean(boolean, Span::initial()) 154 | } 155 | } 156 | 157 | impl From for Literal { 158 | fn from(float: f64) -> Self { 159 | Literal::Float(float, Span::initial()) 160 | } 161 | } 162 | 163 | impl From for Literal { 164 | fn from(int: i64) -> Self { 165 | Literal::Integer(int, Span::initial()) 166 | } 167 | } 168 | 169 | impl<'a> From<&'a Path> for Literal { 170 | fn from(path: &'a Path) -> Self { 171 | Literal::Path(path.to_owned(), Span::initial()) 172 | } 173 | } 174 | 175 | impl From for Literal { 176 | fn from(path: PathBuf) -> Self { 177 | Literal::Path(path, Span::initial()) 178 | } 179 | } 180 | 181 | impl From<((), S)> for Literal { 182 | fn from((_, span): ((), S)) -> Self { 183 | Literal::Null(span.to_span()) 184 | } 185 | } 186 | 187 | impl From<(bool, S)> for Literal { 188 | fn from((boolean, span): (bool, S)) -> Self { 189 | Literal::Boolean(boolean, span.to_span()) 190 | } 191 | } 192 | 193 | impl From<(f64, S)> for Literal { 194 | fn from((float, span): (f64, S)) -> Self { 195 | Literal::Float(float, span.to_span()) 196 | } 197 | } 198 | 199 | impl From<(i64, S)> for Literal { 200 | fn from((int, span): (i64, S)) -> Self { 201 | Literal::Integer(int, span.to_span()) 202 | } 203 | } 204 | 205 | impl<'a, S: ToSpan> From<(&'a Path, S)> for Literal { 206 | fn from((path, span): (&'a Path, S)) -> Self { 207 | Literal::Path(path.to_owned(), span.to_span()) 208 | } 209 | } 210 | 211 | impl From<(PathBuf, S)> for Literal { 212 | fn from((path, span): (PathBuf, S)) -> Self { 213 | Literal::Path(path, span.to_span()) 214 | } 215 | } 216 | 217 | impl From<(Url, S)> for Literal { 218 | fn from((uri, span): (Url, S)) -> Self { 219 | Literal::Uri(uri, span.to_span()) 220 | } 221 | } 222 | 223 | impl HasSpan for Literal { 224 | fn span(&self) -> Span { 225 | match *self { 226 | Literal::Null(ref e) => *e, 227 | Literal::Boolean(_, ref e) => *e, 228 | Literal::Float(_, ref e) => *e, 229 | Literal::Integer(_, ref e) => *e, 230 | Literal::Path(_, ref e) => *e, 231 | Literal::PathTemplate(_, ref e) => *e, 232 | Literal::Uri(_, ref e) => *e, 233 | } 234 | } 235 | } 236 | 237 | impl PartialEq for Literal { 238 | fn eq(&self, other: &Self) -> bool { 239 | use Literal::*; 240 | match (self, other) { 241 | (Null(_), Null(_)) => true, 242 | (Boolean(ref b1, _), Boolean(ref b2, _)) => *b1 == *b2, 243 | (Float(ref f1, _), Float(ref f2, _)) => f1 == f2, 244 | (Integer(ref i1, _), Integer(ref i2, _)) => i1 == i2, 245 | (Path(ref p1, _), Path(ref p2, _)) => p1 == p2, 246 | (PathTemplate(ref p1, _), PathTemplate(ref p2, _)) => p1 == p2, 247 | (Uri(ref u1, _), Uri(ref u2, _)) => u1 == u2, 248 | _ => false, 249 | } 250 | } 251 | } 252 | 253 | impl PartialOrd for Literal { 254 | fn partial_cmp(&self, other: &Self) -> Option { 255 | use Literal::*; 256 | match (self, other) { 257 | (Null(_), Null(_)) => Some(Ordering::Equal), 258 | (Boolean(ref b1, _), Boolean(ref b2, _)) => b1.partial_cmp(b2), 259 | (Float(ref f1, _), Float(ref f2, _)) => f1.partial_cmp(f2), 260 | (Integer(ref i1, _), Integer(ref i2, _)) => i1.partial_cmp(i2), 261 | (Path(ref p1, _), Path(ref p2, _)) => p1.partial_cmp(p2), 262 | (PathTemplate(ref p1, _), PathTemplate(ref p2, _)) => p1.partial_cmp(p2), 263 | (Uri(ref u1, _), Uri(ref u2, _)) => u1.to_string().partial_cmp(&u2.to_string()), 264 | _ => None, 265 | } 266 | } 267 | } 268 | -------------------------------------------------------------------------------- /nix-parser/src/error.rs: -------------------------------------------------------------------------------- 1 | //! Error reporting data structures. 2 | 3 | pub use self::expected_found::ExpectedFoundError; 4 | pub use self::incorrect_delim::IncorrectDelimError; 5 | pub use self::unclosed_delim::UnclosedDelimError; 6 | pub use self::unexpected::UnexpectedError; 7 | 8 | use std::borrow::Cow; 9 | use std::fmt::{Display, Formatter, Result as FmtResult}; 10 | use std::iter::FromIterator; 11 | use std::slice::Iter; 12 | 13 | use codespan::{FileId, Span}; 14 | use codespan_reporting::diagnostic::{Diagnostic, Label}; 15 | use nom::error::{ErrorKind, ParseError}; 16 | use smallvec::{smallvec, IntoIter, SmallVec}; 17 | 18 | use crate::ToSpan; 19 | 20 | mod expected_found; 21 | mod incorrect_delim; 22 | mod unclosed_delim; 23 | mod unexpected; 24 | 25 | /// Trait for converting error types to pretty-printable diagnostics. 26 | /// 27 | /// # Examples 28 | /// 29 | /// ``` 30 | /// use codespan::{Files, FileId, Span}; 31 | /// use codespan_reporting::diagnostic::{Diagnostic, Label}; 32 | /// use nix_parser::error::ToDiagnostic; 33 | /// 34 | /// struct MyError; 35 | /// 36 | /// impl ToDiagnostic for MyError { 37 | /// fn to_diagnostic(&self, file: FileId) -> Diagnostic { 38 | /// let label = Label::new(file, Span::new(2, 3), "error occurred here"); 39 | /// Diagnostic::new_error("something went wrong", label) 40 | /// } 41 | /// } 42 | /// 43 | /// let mut files = Files::new(); 44 | /// let file_id = files.add("example.nix", "1 + 1"); 45 | /// 46 | /// let error = MyError; 47 | /// println!("{:?}", error.to_diagnostic(file_id)); 48 | /// ``` 49 | pub trait ToDiagnostic { 50 | /// Converts this type to a [`Diagnostic`] using the given file ID. 51 | /// 52 | /// [`Diagnostic`]: https://docs.rs/codespan-reporting/0.5.0/codespan_reporting/diagnostic/struct.Diagnostic.html 53 | fn to_diagnostic(&self, file: FileId) -> Diagnostic; 54 | } 55 | 56 | /// A growable stack for accumulating errors. 57 | #[derive(Clone, Debug, Eq, PartialEq)] 58 | pub struct Errors { 59 | errors: SmallVec<[Error; 1]>, 60 | } 61 | 62 | impl Errors { 63 | /// Constructs a new, empty `Errors` stack. 64 | /// 65 | /// The stack will not allocate until new errors are pushed onto it. 66 | /// 67 | /// # Examples 68 | /// 69 | /// ``` 70 | /// # #![allow(unused_mut)] 71 | /// # use nix_parser::error::Errors; 72 | /// let mut errors = Errors::new(); 73 | /// ``` 74 | #[inline] 75 | pub fn new() -> Self { 76 | Errors { 77 | errors: SmallVec::new(), 78 | } 79 | } 80 | 81 | /// Returns the number of errors in the stack. 82 | /// 83 | /// # Examples 84 | /// 85 | /// ``` 86 | /// # use nix_parser::error::Errors; 87 | /// let errors = Errors::new(); 88 | /// assert_eq!(errors.len(), 0); 89 | /// ``` 90 | #[inline] 91 | pub fn len(&self) -> usize { 92 | self.errors.len() 93 | } 94 | 95 | /// Returns `true` if the error stack is empty. 96 | /// 97 | /// # Examples 98 | /// 99 | /// ``` 100 | /// # use nix_parser::error::{Errors, UnexpectedError}; 101 | /// use codespan::Span; 102 | /// 103 | /// let mut errors = Errors::new(); 104 | /// assert!(errors.is_empty()); 105 | /// 106 | /// errors.push(UnexpectedError::new("token", Span::new(3, 4))); 107 | /// assert!(!errors.is_empty()); 108 | /// ``` 109 | #[inline] 110 | pub fn is_empty(&self) -> bool { 111 | self.errors.is_empty() 112 | } 113 | 114 | /// Appends a new error to the stack. 115 | /// 116 | /// # Examples 117 | /// 118 | /// ``` 119 | /// # use nix_parser::error::{Errors, ExpectedFoundError}; 120 | /// use codespan::Span; 121 | /// 122 | /// let mut errors = Errors::new(); 123 | /// errors.push(ExpectedFoundError::new("foo", "bar", Span::new(0, 4))); 124 | /// assert_eq!(errors.len(), 1); 125 | /// ``` 126 | #[inline] 127 | pub fn push(&mut self, error: E) 128 | where 129 | E: Into, 130 | { 131 | self.errors.push(error.into()); 132 | } 133 | 134 | /// Removes the last error from the stack and returns it, or [`None`] if it is empty. 135 | /// 136 | /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None 137 | /// 138 | /// # Examples 139 | /// 140 | /// ``` 141 | /// # use nix_parser::error::{Error, Errors, UnexpectedError}; 142 | /// use codespan::Span; 143 | /// 144 | /// let mut errors = Errors::new(); 145 | /// assert_eq!(errors.pop(), None); 146 | /// 147 | /// errors.push(UnexpectedError::new("token", Span::new(3, 4))); 148 | /// assert_eq!(errors.pop(), Some(Error::Unexpected(UnexpectedError::new("token", Span::new(3, 4))))); 149 | /// ``` 150 | #[inline] 151 | pub fn pop(&mut self) -> Option { 152 | self.errors.pop() 153 | } 154 | 155 | /// Returns the last error in the stack, or [`None`] if it is empty. 156 | /// 157 | /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None 158 | /// 159 | /// # Examples 160 | /// 161 | /// ``` 162 | /// # use nix_parser::error::{Error, Errors, UnexpectedError}; 163 | /// use codespan::Span; 164 | /// 165 | /// let mut empty = Errors::new(); 166 | /// assert_eq!(empty.last(), None); 167 | /// 168 | /// let mut one = Errors::new(); 169 | /// one.push(UnexpectedError::new("token", Span::new(3, 4))); 170 | /// assert_eq!(one.last(), Some(&Error::Unexpected(UnexpectedError::new("token", Span::new(3, 4))))); 171 | /// ``` 172 | #[inline] 173 | pub fn last(&mut self) -> Option<&Error> { 174 | self.errors.last() 175 | } 176 | 177 | /// Returns an iterator of errors. 178 | /// 179 | /// # Examples 180 | /// 181 | /// ``` 182 | /// # use nix_parser::error::{Error, Errors, ExpectedFoundError, UnexpectedError}; 183 | /// use codespan::Span; 184 | /// 185 | /// let mut errors = Errors::new(); 186 | /// errors.push(UnexpectedError::new("token", Span::new(3, 4))); 187 | /// errors.push(ExpectedFoundError::new("foo", "bar", Span::new(0, 4))); 188 | /// 189 | /// let mut iter = errors.iter(); 190 | /// assert_eq!(iter.next(), Some(&Error::Unexpected(UnexpectedError::new("token", Span::new(3, 4))))); 191 | /// assert_eq!(iter.next(), Some(&Error::ExpectedFound(ExpectedFoundError::new("foo", "bar", Span::new(0, 4))))); 192 | /// assert_eq!(iter.next(), None); 193 | /// ``` 194 | #[inline] 195 | pub fn iter(&self) -> Iter { 196 | self.errors.iter() 197 | } 198 | 199 | /// Converts each error to a new [`Diagnostic`] and collects them in a [`Vec`]. 200 | /// 201 | /// [`Diagnostic`]: https://docs.rs/codespan-reporting/0.5.0/codespan_reporting/diagnostic/struct.Diagnostic.html 202 | /// [`Vec`]: https://doc.rust-lang.org/std/vec/struct.Vec.html 203 | /// 204 | /// # Examples 205 | /// 206 | /// ``` 207 | /// # use nix_parser::error::{Errors, ExpectedFoundError}; 208 | /// use codespan::{Files, FileId, Span}; 209 | /// 210 | /// let mut files = Files::new(); 211 | /// let file_id = files.add("example.nix", "1 + 1"); 212 | /// 213 | /// let mut errors = Errors::new(); 214 | /// errors.push(ExpectedFoundError::new("-", "+", Span::new(2, 2))); 215 | /// 216 | /// let diagnostics = errors.to_diagnostics(file_id); 217 | /// println!("{:?}", diagnostics); 218 | /// ``` 219 | pub fn to_diagnostics(&self, file: FileId) -> Vec { 220 | self.errors.iter().map(|e| e.to_diagnostic(file)).collect() 221 | } 222 | } 223 | 224 | impl Default for Errors { 225 | /// Creates an empty `Errors` stack. 226 | #[inline] 227 | fn default() -> Self { 228 | Errors::new() 229 | } 230 | } 231 | 232 | impl Display for Errors { 233 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 234 | let mut errors = self.errors.iter().enumerate(); 235 | 236 | if let Some((i, error)) = errors.next() { 237 | write!(fmt, "{}: {}", i, error)?; 238 | } 239 | 240 | for (i, error) in errors { 241 | write!(fmt, "\n{}: {}", i, error)?; 242 | } 243 | 244 | Ok(()) 245 | } 246 | } 247 | 248 | impl std::error::Error for Errors {} 249 | 250 | impl Extend for Errors { 251 | fn extend(&mut self, iter: I) 252 | where 253 | I: IntoIterator, 254 | { 255 | self.errors.extend(iter); 256 | } 257 | } 258 | 259 | impl FromIterator for Errors { 260 | fn from_iter(iter: I) -> Self 261 | where 262 | I: IntoIterator, 263 | { 264 | Errors { 265 | errors: SmallVec::from_iter(iter), 266 | } 267 | } 268 | } 269 | 270 | impl IntoIterator for Errors { 271 | type Item = Error; 272 | type IntoIter = IntoIter<[Self::Item; 1]>; 273 | 274 | #[inline] 275 | fn into_iter(self) -> Self::IntoIter { 276 | self.errors.into_iter() 277 | } 278 | } 279 | 280 | impl<'a> IntoIterator for &'a Errors { 281 | type Item = &'a Error; 282 | type IntoIter = Iter<'a, Error>; 283 | 284 | #[inline] 285 | fn into_iter(self) -> Self::IntoIter { 286 | self.errors.iter() 287 | } 288 | } 289 | 290 | /// Implemented so `Errors` can be used as a custom error type in a [`nom::IResult`]. 291 | /// 292 | /// [`nom::IResult`]: https://docs.rs/nom/5.0.1/nom/type.IResult.html 293 | impl ParseError for Errors 294 | where 295 | I: ToSpan + ToString, 296 | { 297 | fn from_error_kind(input: I, kind: ErrorKind) -> Self { 298 | Errors { 299 | errors: smallvec![Error::Nom(input.to_span(), kind)], 300 | } 301 | } 302 | 303 | fn append(input: I, kind: ErrorKind, mut other: Self) -> Self { 304 | if cfg!(debug_assertions) { 305 | other.push(Error::Nom(input.to_span(), kind)); 306 | } 307 | other 308 | } 309 | } 310 | 311 | /// Kinds of errors that can accumulate in an [`Errors`] stack during parsing. 312 | /// 313 | /// [`Errors`]: ./struct.Error.html 314 | /// 315 | /// This error type implements [`ToDiagnostic`] so it can be easily converted to a pretty-printable 316 | /// [`Diagnostic`]. 317 | /// 318 | /// [`ToDiagnostic`]: ./trait.ToDiagnostic.html 319 | /// [`Diagnostic`]: https://docs.rs/codespan-reporting/0.5.0/codespan_reporting/diagnostic/struct.Diagnostic.html 320 | #[derive(Clone, Debug, Eq, PartialEq)] 321 | pub enum Error { 322 | /// A certain item was found, but was expecting something else. 323 | ExpectedFound(ExpectedFoundError), 324 | /// An incorrect closing delimiter was specified. 325 | IncorrectDelim(IncorrectDelimError), 326 | /// At least one delimited span was left unclosed. 327 | UnclosedDelim(UnclosedDelimError), 328 | /// An unexpected token was found. 329 | Unexpected(UnexpectedError), 330 | /// A custom error with a span and message. 331 | Message(Span, Cow<'static, str>), 332 | /// A [`nom`] parse error occurred. 333 | /// 334 | /// [`nom`]: https://docs.rs/nom/5.0.1/nom/ 335 | /// 336 | /// This kind of error may occur during parsing, but is expected to be discarded immediately 337 | /// once a successful path is found. Such errors should not normally be displayed to the user, 338 | /// as it indicates an unhandled case in the parser. 339 | Nom(Span, ErrorKind), 340 | } 341 | 342 | impl Display for Error { 343 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 344 | match *self { 345 | Error::ExpectedFound(ref e) => write!(fmt, "{}", e), 346 | Error::IncorrectDelim(ref e) => write!(fmt, "{}", e), 347 | Error::UnclosedDelim(ref e) => write!(fmt, "{}", e), 348 | Error::Unexpected(ref e) => write!(fmt, "{}", e), 349 | Error::Message(_, ref e) => write!(fmt, "{}", e), 350 | Error::Nom(_, ref e) => write!(fmt, "nom error: {:?}", e), 351 | } 352 | } 353 | } 354 | 355 | impl std::error::Error for Error {} 356 | 357 | impl From for Error { 358 | fn from(error: ExpectedFoundError) -> Self { 359 | Error::ExpectedFound(error) 360 | } 361 | } 362 | 363 | impl From for Error { 364 | fn from(error: IncorrectDelimError) -> Self { 365 | Error::IncorrectDelim(error) 366 | } 367 | } 368 | 369 | impl From for Error { 370 | fn from(error: UnclosedDelimError) -> Self { 371 | Error::UnclosedDelim(error) 372 | } 373 | } 374 | 375 | impl From for Error { 376 | fn from(error: UnexpectedError) -> Self { 377 | Error::Unexpected(error) 378 | } 379 | } 380 | 381 | impl ToDiagnostic for Error { 382 | fn to_diagnostic(&self, file: FileId) -> Diagnostic { 383 | match *self { 384 | Error::ExpectedFound(ref e) => e.to_diagnostic(file), 385 | Error::IncorrectDelim(ref e) => e.to_diagnostic(file), 386 | Error::UnclosedDelim(ref e) => e.to_diagnostic(file), 387 | Error::Unexpected(ref e) => e.to_diagnostic(file), 388 | Error::Message(ref span, ref msg) => { 389 | let label = Label::new(file, *span, msg.clone()); 390 | Diagnostic::new_error(msg.clone(), label) 391 | } 392 | Error::Nom(ref span, ref kind) => { 393 | let label = Label::new(file, *span, self.to_string()); 394 | let note = "note: this indicates an unhandled case in the parser".to_string(); 395 | Diagnostic::new_bug(format!("nom error: {:?}", kind), label).with_notes(vec![note]) 396 | } 397 | } 398 | } 399 | } 400 | -------------------------------------------------------------------------------- /nix-parser/src/error/expected_found.rs: -------------------------------------------------------------------------------- 1 | //! Expected/found error data structure. 2 | 3 | use std::borrow::Cow; 4 | use std::error::Error; 5 | use std::fmt::{Display, Formatter, Result as FmtResult}; 6 | 7 | use codespan::{FileId, Span}; 8 | use codespan_reporting::diagnostic::{Diagnostic, Label}; 9 | 10 | use super::ToDiagnostic; 11 | use crate::ToSpan; 12 | 13 | /// Error that occurs when an item was found, but was expecting something else. 14 | #[derive(Clone, Debug, Eq, PartialEq)] 15 | pub struct ExpectedFoundError { 16 | /// Printable name of the item that was expected. 17 | pub expected: Cow<'static, str>, 18 | /// Printable name of the item that was found. 19 | pub found: Cow<'static, str>, 20 | /// Span of the found item. 21 | pub span: Span, 22 | } 23 | 24 | impl ExpectedFoundError { 25 | /// Constructs a new `ExpectedFoundError`. 26 | pub fn new(expected: T, found: U, span: S) -> Self 27 | where 28 | T: Into>, 29 | U: Into>, 30 | S: ToSpan, 31 | { 32 | ExpectedFoundError { 33 | expected: expected.into(), 34 | found: found.into(), 35 | span: span.to_span(), 36 | } 37 | } 38 | } 39 | 40 | impl Display for ExpectedFoundError { 41 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 42 | write!(fmt, "expected {}, found {}", self.expected, self.found) 43 | } 44 | } 45 | 46 | impl Error for ExpectedFoundError {} 47 | 48 | impl ToDiagnostic for ExpectedFoundError { 49 | fn to_diagnostic(&self, file: FileId) -> Diagnostic { 50 | let label = Label::new(file, self.span, format!("expected {} here", self.expected)); 51 | Diagnostic::new_error(self.to_string(), label) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /nix-parser/src/error/incorrect_delim.rs: -------------------------------------------------------------------------------- 1 | //! Incorrect closing delimiter error data structure. 2 | 3 | use std::error::Error; 4 | use std::fmt::{Display, Formatter, Result as FmtResult}; 5 | 6 | use codespan::{FileId, Span}; 7 | use codespan_reporting::diagnostic::{Diagnostic, Label}; 8 | 9 | use super::ToDiagnostic; 10 | use crate::ToSpan; 11 | 12 | /// Error that occurs when an incorrect closing delimiter was specified. 13 | #[derive(Clone, Debug, Eq, PartialEq)] 14 | pub struct IncorrectDelimError { 15 | /// The unmatched delimiter and its location in the source file. 16 | pub unmatched_delim: (char, Span), 17 | /// Location where a possible closing delimiter could be placed. 18 | pub candidate_span: Option, 19 | /// Span from the unmatched character to EOF. 20 | pub unclosed_span: Option, 21 | } 22 | 23 | impl IncorrectDelimError { 24 | /// Constructs a new `IncorrectDelimError`. 25 | pub fn new(delim: char, span: S, candidate: Option, unclosed: Option) -> Self 26 | where 27 | S: ToSpan, 28 | { 29 | IncorrectDelimError { 30 | unmatched_delim: (delim, span.to_span()), 31 | candidate_span: candidate.map(|s| s.to_span()), 32 | unclosed_span: unclosed.map(|s| s.to_span()), 33 | } 34 | } 35 | } 36 | 37 | impl Display for IncorrectDelimError { 38 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 39 | write!( 40 | fmt, 41 | "incorrect close delimiter: `{}`", 42 | self.unmatched_delim.0 43 | ) 44 | } 45 | } 46 | 47 | impl Error for IncorrectDelimError {} 48 | 49 | impl ToDiagnostic for IncorrectDelimError { 50 | fn to_diagnostic(&self, file: FileId) -> Diagnostic { 51 | let primary = Label::new(file, self.unmatched_delim.1, "incorrect close delimiter"); 52 | let mut diagnostic = Diagnostic::new_error(self.to_string(), primary); 53 | 54 | if let Some(span) = self.candidate_span { 55 | let candidate = Label::new(file, span, "close delimiter possibly meant for this"); 56 | diagnostic.secondary_labels.push(candidate); 57 | } 58 | 59 | if let Some(span) = self.unclosed_span { 60 | let unclosed = Label::new(file, span, "unmatched delimiter"); 61 | diagnostic.secondary_labels.push(unclosed); 62 | } 63 | 64 | diagnostic 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /nix-parser/src/error/unclosed_delim.rs: -------------------------------------------------------------------------------- 1 | /// Unclosed delimiters error data structure. 2 | use std::error::Error; 3 | use std::fmt::{Display, Formatter, Result as FmtResult}; 4 | 5 | use codespan::{FileId, Span}; 6 | use codespan_reporting::diagnostic::{Diagnostic, Label}; 7 | 8 | use super::ToDiagnostic; 9 | use crate::ToSpan; 10 | 11 | /// Error that occurs when at least one delimited span was left unclosed. 12 | #[derive(Clone, Debug, Eq, PartialEq)] 13 | pub struct UnclosedDelimError { 14 | /// Locations of open delimiters that lack a matching close delimiter. 15 | pub unclosed_delims: Vec, 16 | /// Span pointing to the end of the file. 17 | pub eof_span: Span, 18 | } 19 | 20 | impl UnclosedDelimError { 21 | /// Constructs a new `UnclosedDelimError`. 22 | pub fn new(delims: Vec, eof_span: S2) -> Self 23 | where 24 | S1: ToSpan, 25 | S2: ToSpan, 26 | { 27 | UnclosedDelimError { 28 | unclosed_delims: delims.into_iter().map(|span| span.to_span()).collect(), 29 | eof_span: eof_span.to_span(), 30 | } 31 | } 32 | } 33 | 34 | impl Display for UnclosedDelimError { 35 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 36 | write!(fmt, "this file contains un-closed delimiters") 37 | } 38 | } 39 | 40 | impl Error for UnclosedDelimError {} 41 | 42 | impl ToDiagnostic for UnclosedDelimError { 43 | fn to_diagnostic(&self, file: FileId) -> Diagnostic { 44 | let primary = Label::new(file, self.eof_span, "expected matching delimiter here"); 45 | let mut diagnostic = Diagnostic::new_error(self.to_string(), primary); 46 | 47 | for span in &self.unclosed_delims { 48 | let unclosed = Label::new(file, *span, "unmatched delimiter"); 49 | diagnostic.secondary_labels.push(unclosed); 50 | } 51 | 52 | diagnostic 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /nix-parser/src/error/unexpected.rs: -------------------------------------------------------------------------------- 1 | //! Unexpected token error data structure. 2 | 3 | use std::borrow::Cow; 4 | use std::error::Error; 5 | use std::fmt::{Display, Formatter, Result as FmtResult}; 6 | 7 | use codespan::{FileId, Span}; 8 | use codespan_reporting::diagnostic::{Diagnostic, Label}; 9 | 10 | use super::ToDiagnostic; 11 | use crate::ToSpan; 12 | 13 | /// Error that occurs when an unexpected token was found. 14 | #[derive(Clone, Debug, Eq, PartialEq)] 15 | pub struct UnexpectedError { 16 | /// Printable name of the token that was found. 17 | pub token: Cow<'static, str>, 18 | /// Span of the found token. 19 | pub span: Span, 20 | } 21 | 22 | impl UnexpectedError { 23 | /// Constructs a new `UnexpectedError`. 24 | pub fn new(token: T, span: S) -> Self 25 | where 26 | T: Into>, 27 | S: ToSpan, 28 | { 29 | UnexpectedError { 30 | token: token.into(), 31 | span: span.to_span(), 32 | } 33 | } 34 | } 35 | 36 | impl Display for UnexpectedError { 37 | fn fmt(&self, fmt: &mut Formatter) -> FmtResult { 38 | write!(fmt, "unexpected token: {}", self.token) 39 | } 40 | } 41 | 42 | impl Error for UnexpectedError {} 43 | 44 | impl ToDiagnostic for UnexpectedError { 45 | fn to_diagnostic(&self, file: FileId) -> Diagnostic { 46 | let label = Label::new(file, self.span, "found unexpected token here"); 47 | Diagnostic::new_error(self.to_string(), label) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /nix-parser/src/lexer.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for lexing and tokenizing. 2 | 3 | pub use self::tokens::{CommentKind, StringFragment, StringKind, Token, Tokens}; 4 | 5 | use codespan::Span; 6 | use nom::branch::alt; 7 | use nom::bytes::complete::take; 8 | use nom::character::complete::multispace1; 9 | use nom::combinator::{iterator, map}; 10 | use nom::error::ErrorKind; 11 | 12 | use self::lexers::{comment, identifier, interpolation, literal, operator, punctuation, string}; 13 | use self::util::check_delims_balanced; 14 | use crate::error::{Error, Errors, UnexpectedError}; 15 | use crate::ToSpan; 16 | 17 | mod lexers; 18 | mod tokens; 19 | mod util; 20 | 21 | type LocatedSpan<'a> = nom_locate::LocatedSpan<&'a str>; 22 | type IResult<'a, T> = nom::IResult, T>; 23 | 24 | impl<'a> ToSpan for LocatedSpan<'a> { 25 | fn to_span(&self) -> Span { 26 | let start = self.offset; 27 | let end = start + self.fragment.len(); 28 | Span::new(start as u32, end as u32) 29 | } 30 | } 31 | 32 | /// Converts an input string into a sequence of tokens. 33 | /// 34 | /// # Examples 35 | /// 36 | /// ``` 37 | /// # use nix_parser::error::Errors; 38 | /// # fn main() -> Result<(), Errors> { 39 | /// use nix_parser::lexer::Lexer; 40 | /// 41 | /// let expr = r#" 42 | /// { foo, bar, ... }: 43 | /// with import {}; 44 | /// let 45 | /// inherit foo; 46 | /// baz = { quux = 12 + bar; }; 47 | /// in 48 | /// { inherit foo baz; } 49 | /// "#; 50 | /// 51 | /// let lexer = Lexer::new(expr)?; 52 | /// println!("produced tokens: {:?}", lexer.tokens()); 53 | /// println!("lexing errors: {:?}", lexer.errors()); 54 | /// # Ok(()) 55 | /// # } 56 | /// ``` 57 | #[derive(Clone, Debug, PartialEq)] 58 | pub struct Lexer<'a> { 59 | tokens: Vec>, 60 | errors: Errors, 61 | } 62 | 63 | impl<'a> Lexer<'a> { 64 | /// Constructs a new `Lexer` and converts the source text into a sequence of tokens. 65 | /// 66 | /// This constructor strips all [`Token::Whitespace`] tokens out of the text. 67 | /// 68 | /// [`Token::Whitespace`]: ./enum.Token.html#variant.Whitespace 69 | /// 70 | /// Once the text has been tokenized, all tokens of type [`Token::Unknown`] will be stripped 71 | /// out and converted to errors and delimiter balancing is checked. Errors are produced during 72 | /// these two phases are not fatal, though, and are accessible via the [`errors`] method. 73 | /// 74 | /// [`Token::Unknown`]: ./enum.Token.html#variant.Unknown 75 | /// [`errors`]: #method.errors 76 | /// 77 | /// The tokenization process will only return `Err` if the source text is empty or consists 78 | /// only of comments or invalid tokens, or if the lexer itself has a bug. 79 | pub fn new(s: &'a str) -> Result { 80 | Self::new_filtered(s, |token| !token.is_whitespace()) 81 | } 82 | 83 | /// Constructs a new `Lexer` and converts the source text into a sequence of tokens. 84 | /// 85 | /// This constructor preserves all [`Token::Whitespace`] tokens. 86 | /// 87 | /// [`Token::Whitespace`]: ./enum.Token.html#variant.Whitespace 88 | /// 89 | /// Once the text has been tokenized, all tokens of type [`Token::Unknown`] will be stripped 90 | /// out and converted to errors and delimiter balancing is checked. Errors are produced during 91 | /// these two phases are not fatal, though, and are accessible via the [`errors`] method. 92 | /// 93 | /// [`Token::Unknown`]: ./enum.Token.html#variant.Unknown 94 | /// [`errors`]: #method.errors 95 | /// 96 | /// The tokenization process will only return `Err` if the source text is empty or consists 97 | /// only of comments or invalid tokens, or if the lexer itself has a bug. 98 | pub fn new_with_whitespace(s: &'a str) -> Result { 99 | Self::new_filtered(s, |_| true) 100 | } 101 | 102 | fn new_filtered(s: &'a str, f: F) -> Result 103 | where 104 | F: Fn(&Token) -> bool, 105 | { 106 | let input = LocatedSpan::new(s); 107 | let mut lexer = iterator(input, token); 108 | let (mut tokens, mut errors) = filter_unexpected_tokens(lexer.filter(f)); 109 | 110 | match lexer.finish() { 111 | Ok((remaining, _)) if remaining.fragment.is_empty() => { 112 | let end = input.fragment.len().saturating_sub(1) as u32; 113 | let eof_span = Span::new(end, end); 114 | 115 | let only_trivia = tokens.iter().all(|t| t.is_trivia()); 116 | let errors = if tokens.is_empty() || only_trivia { 117 | let message = "Nix expressions must resolve to a value".into(); 118 | errors.push(Error::Message(Span::initial(), message)); 119 | return Err(errors); 120 | } else { 121 | errors.extend(check_delims_balanced(&tokens, eof_span)); 122 | errors 123 | }; 124 | 125 | tokens.push(Token::Eof(eof_span)); 126 | Ok(Lexer { tokens, errors }) 127 | } 128 | Ok((remaining, _)) => { 129 | let mut errors = Errors::new(); 130 | errors.push(Error::Nom(remaining.to_span(), ErrorKind::Eof)); 131 | Err(errors) 132 | } 133 | Err(nom::Err::Error(err)) | Err(nom::Err::Failure(err)) => { 134 | let (input, kind) = err; 135 | let mut errors = Errors::new(); 136 | errors.push(Error::Nom(input.to_span(), kind)); 137 | Err(errors) 138 | } 139 | Err(nom::Err::Incomplete(needed)) => { 140 | let mut errors = Errors::new(); 141 | let message = format!("unable to recover from incomplete input: {:?}", needed); 142 | errors.push(Error::Message(Span::initial(), message.into())); 143 | Err(errors) 144 | } 145 | } 146 | } 147 | 148 | /// Returns a slice over the tokens produced by the `Lexer`. 149 | /// 150 | /// This slice is guaranteed not to include any tokens that are [`Token::Unknown`], as these 151 | /// are converted to errors accessible via the [`errors`] method. 152 | /// 153 | /// [`Token::Unknown`]: ./enum.Token.html#variant.Unknown 154 | /// [`errors`]: #method.errors 155 | /// 156 | /// # Examples 157 | /// 158 | /// ``` 159 | /// # use nix_parser::error::Errors; 160 | /// # use nix_parser::lexer::Lexer; 161 | /// # fn main() -> Result<(), Errors> { 162 | /// use codespan::Span; 163 | /// use nix_parser::lexer::Token; 164 | /// 165 | /// let expr = "{ foo = /nix/store/xrnya64zahcibg89vqhf9cwm5gcnaa6q-rust; }"; 166 | /// let tokens = [ 167 | /// Token::LBrace(Span::new(0, 1)), 168 | /// Token::Identifier("foo".into(), Span::new(2, 5)), 169 | /// Token::Eq(Span::new(6, 7)), 170 | /// Token::Path("/nix/store/xrnya64zahcibg89vqhf9cwm5gcnaa6q-rust".into(), Span::new(8, 56)), 171 | /// Token::Semi(Span::new(56, 57)), 172 | /// Token::RBrace(Span::new(58, 59)), 173 | /// Token::Eof(Span::new(58, 58)), 174 | /// ]; 175 | /// 176 | /// let lexer = Lexer::new(expr)?; 177 | /// assert_eq!(lexer.tokens(), &tokens[..]); 178 | /// # Ok(()) 179 | /// # } 180 | /// ``` 181 | pub fn tokens(&self) -> Tokens<'_> { 182 | Tokens::new(self.tokens.as_slice()) 183 | } 184 | 185 | /// Returns the errors that occurred at tokenization time, if any. 186 | /// 187 | /// Examples of errors that can occur may include unbalanced delimiters, incorrect delimiters, 188 | /// the presence of invalid or unknown tokens, path values with trailing slashes, or 189 | /// unterminated strings or interpolations. 190 | /// 191 | /// # Examples 192 | /// 193 | /// ``` 194 | /// # use nix_parser::error::Errors; 195 | /// # use nix_parser::lexer::Lexer; 196 | /// # fn main() -> Result<(), Errors> { 197 | /// use codespan::Span; 198 | /// 199 | /// let lexer = Lexer::new("{ % is not allowed")?; 200 | /// 201 | /// // Two lexing errors: there is an unclosed `{` delimiter, and `%` is an unknown character. 202 | /// assert_eq!(lexer.errors().len(), 2); 203 | /// # Ok(()) 204 | /// # } 205 | /// ``` 206 | pub fn errors(&self) -> &Errors { 207 | &self.errors 208 | } 209 | } 210 | 211 | fn token(input: LocatedSpan) -> IResult { 212 | alt(( 213 | whitespace, 214 | literal, 215 | identifier, 216 | string, 217 | interpolation, 218 | comment, 219 | punctuation, 220 | operator, 221 | unknown, 222 | ))(input) 223 | } 224 | 225 | fn whitespace(input: LocatedSpan) -> IResult { 226 | map(multispace1, |span: LocatedSpan| { 227 | Token::Whitespace(span.to_span()) 228 | })(input) 229 | } 230 | 231 | fn unknown(input: LocatedSpan) -> IResult { 232 | map(take(1usize), |span: LocatedSpan| { 233 | let error = UnexpectedError::new(format!("`{}`", span.fragment), span.to_span()); 234 | Token::Unknown(span.fragment.into(), span.to_span(), error.into()) 235 | })(input) 236 | } 237 | 238 | fn filter_unexpected_tokens<'a, I>(tokens: I) -> (Vec>, Errors) 239 | where 240 | I: Iterator>, 241 | { 242 | let mut valid = Vec::new(); 243 | let mut errors = Errors::new(); 244 | 245 | for token in tokens { 246 | match token { 247 | Token::Unknown(_, _, error) => errors.push(error), 248 | token => valid.push(token), 249 | } 250 | } 251 | 252 | (valid, errors) 253 | } 254 | -------------------------------------------------------------------------------- /nix-parser/src/lexer/lexers.rs: -------------------------------------------------------------------------------- 1 | //! Combinators for splitting a source text into tokens. 2 | 3 | pub use self::string::string; 4 | 5 | use codespan::Span; 6 | use nom::branch::alt; 7 | use nom::bytes::complete::{tag, take_while, take_while1}; 8 | use nom::character::complete::{anychar, char, line_ending, multispace0, not_line_ending, space0}; 9 | use nom::combinator::{map, peek, recognize, verify}; 10 | use nom::multi::separated_nonempty_list; 11 | use nom::sequence::{pair, preceded, tuple}; 12 | use nom::Slice; 13 | use once_cell::sync::OnceCell; 14 | use regex::Regex; 15 | 16 | use self::number::{float, integer}; 17 | use self::path::{path, path_template}; 18 | use self::uri::uri; 19 | use super::util::{map_spanned, split_lines_without_indent}; 20 | use super::{token, CommentKind, IResult, LocatedSpan, Token}; 21 | use crate::error::Error; 22 | use crate::ToSpan; 23 | 24 | mod number; 25 | mod path; 26 | mod string; 27 | mod uri; 28 | 29 | pub fn comment(input: LocatedSpan) -> IResult { 30 | let span = map(not_line_ending, |s: LocatedSpan| s.fragment); 31 | let rows = separated_nonempty_list(pair(line_ending, space0), preceded(char('#'), span)); 32 | let text = map(rows, |rows| rows.join("\n")); 33 | let line_comment = map_spanned(text, |span, t| Token::Comment(t, CommentKind::Line, span)); 34 | alt((line_comment, block_comment))(input) 35 | } 36 | 37 | fn block_comment(input: LocatedSpan) -> IResult { 38 | let close_tag = recognize(pair(take_while1(|c: char| c == '*'), char('/'))); 39 | let maybe_consume_stars = alt((peek(close_tag), take_while(|c: char| c == '*'))); 40 | let (remaining, _) = tuple((tag("/*"), maybe_consume_stars, multispace0))(input)?; 41 | 42 | static REGEX: OnceCell = OnceCell::new(); 43 | let regex = REGEX.get_or_init(|| Regex::new(r#"\*+/"#).unwrap()); 44 | 45 | if let Some(m) = regex.find(remaining.fragment) { 46 | let span = remaining.slice(..m.start()); 47 | let remaining = remaining.slice(m.end()..); 48 | let rows: Vec<_> = split_lines_without_indent(span, span.get_column()).collect(); 49 | let comment = Token::Comment(rows.join("\n"), CommentKind::Block, span.to_span()); 50 | Ok((remaining, comment)) 51 | } else { 52 | let end = remaining.fragment.len(); 53 | let remaining = remaining.slice(end.saturating_sub(1)..end); 54 | let error = Error::Message(input.to_span(), "unterminated block comment".into()); 55 | let unknown = Token::Unknown(input.fragment.into(), input.to_span(), error); 56 | Ok((remaining, unknown)) 57 | } 58 | } 59 | 60 | pub fn literal(input: LocatedSpan) -> IResult { 61 | alt((boolean, path, float, integer, path_template, uri))(input) 62 | } 63 | 64 | fn boolean(input: LocatedSpan) -> IResult { 65 | alt(( 66 | map(tag("true"), |s: LocatedSpan| { 67 | Token::Boolean(true, s.to_span()) 68 | }), 69 | map(tag("false"), |s: LocatedSpan| { 70 | Token::Boolean(false, s.to_span()) 71 | }), 72 | ))(input) 73 | } 74 | 75 | pub fn interpolation(input: LocatedSpan) -> IResult { 76 | let (mut remaining, _) = punct_interpolate(input)?; 77 | 78 | let mut tokens = Vec::new(); 79 | let mut depth = 1; 80 | loop { 81 | if let Ok((input, token)) = token(remaining) { 82 | remaining = input; 83 | match token { 84 | Token::LBrace(_) => depth += 1, 85 | Token::RBrace(_) => { 86 | depth -= 1; 87 | if depth == 0 { 88 | break; 89 | } 90 | } 91 | _ => {} 92 | } 93 | tokens.push(token); 94 | } else { 95 | let end = input.fragment.len(); 96 | let remaining = input.slice(end..); 97 | let error = Error::Message(input.to_span(), "unterminated interpolation".into()); 98 | let unknown = Token::Unknown(input.fragment.into(), input.to_span(), error); 99 | return Ok((remaining, unknown)); 100 | } 101 | } 102 | 103 | let span = Span::new(input.offset as u32, remaining.offset as u32); 104 | Ok((remaining, Token::Interpolation(tokens, span))) 105 | } 106 | 107 | macro_rules! define_identifiers { 108 | ($($variant:ident ( $keyword:expr )),+) => { 109 | pub fn identifier(input: LocatedSpan) -> IResult { 110 | let first = verify(anychar, |c: &char| c.is_alphabetic() || *c == '_'); 111 | let rest = take_while(|c: char| c.is_alphanumeric() || "_-'".contains(c)); 112 | let ident = recognize(pair(first, rest)); 113 | map(ident, |span: LocatedSpan| match span.fragment { 114 | $($keyword => Token::$variant(span.to_span()),)+ 115 | frag => Token::Identifier(frag.into(), span.to_span()), 116 | })(input) 117 | } 118 | }; 119 | } 120 | 121 | define_identifiers! { 122 | Assert("assert"), 123 | Else("else"), 124 | If("if"), 125 | In("in"), 126 | Inherit("inherit"), 127 | Let("let"), 128 | Null("null"), 129 | Or("or"), 130 | Rec("rec"), 131 | Then("then"), 132 | With("with") 133 | } 134 | 135 | macro_rules! define_operator { 136 | ($($variant:ident ( $op:expr )),+) => { 137 | pub fn operator(input: LocatedSpan) -> IResult { 138 | alt(($(map(tag($op), |s: LocatedSpan| Token::$variant(s.to_span()))),+))(input) 139 | } 140 | }; 141 | } 142 | 143 | define_operator! { 144 | Concat("++"), 145 | Add("+"), 146 | Imply("->"), 147 | Sub("-"), 148 | Mul("*"), 149 | Update("//"), 150 | Div("/"), 151 | NotEq("!="), 152 | LessThanEq("<="), 153 | LessThan("<"), 154 | GreaterThanEq(">="), 155 | GreaterThan(">"), 156 | LogicalAnd("&&"), 157 | LogicalOr("||"), 158 | Question("?"), 159 | Not("!") 160 | } 161 | 162 | macro_rules! define_punctuation { 163 | ($($function:ident => $variant:ident ( $punct:expr )),+) => { 164 | pub fn punctuation(input: LocatedSpan) -> IResult { 165 | alt(($($function),+))(input) 166 | } 167 | 168 | $( 169 | fn $function(input: LocatedSpan) -> IResult { 170 | map(tag($punct), |s: LocatedSpan| Token::$variant(s.to_span()))(input) 171 | } 172 | )+ 173 | }; 174 | } 175 | 176 | define_punctuation! { 177 | punct_quote_semi => Semi(";"), 178 | punct_comma => Comma(","), 179 | punct_ellipsis => Ellipsis("..."), 180 | punct_dot => Dot("."), 181 | op_eq => IsEq("=="), 182 | punct_eq => Eq("="), 183 | punct_interpolate => Interpolate("${"), 184 | punct_left_brace => LBrace("{"), 185 | punct_right_brace => RBrace("}"), 186 | punct_left_bracket => LBracket("["), 187 | punct_right_bracket => RBracket("]"), 188 | punct_left_paren => LParen("("), 189 | punct_right_paren => RParen(")"), 190 | punct_colon => Colon(":"), 191 | punct_token => At("@") 192 | } 193 | -------------------------------------------------------------------------------- /nix-parser/src/lexer/lexers/number.rs: -------------------------------------------------------------------------------- 1 | //! Combinators for lexing floating-point numbers and integers. 2 | 3 | use nom::branch::alt; 4 | use nom::bytes::complete::is_a; 5 | use nom::character::complete::{char, digit0, digit1, one_of}; 6 | use nom::combinator::{map, opt, recognize}; 7 | use nom::sequence::{pair, tuple}; 8 | 9 | use crate::lexer::{IResult, LocatedSpan, Token}; 10 | use crate::ToSpan; 11 | 12 | pub fn float(input: LocatedSpan) -> IResult { 13 | let first = pair(is_a("123456789"), digit0); 14 | let positive = map(tuple((first, char('.'), digit0)), |_| ()); 15 | let fraction = map(tuple((opt(char('0')), char('.'), digit1)), |_| ()); 16 | let exp = tuple((one_of("Ee"), opt(one_of("+-")), digit1)); 17 | let float = recognize(pair(alt((positive, fraction)), opt(exp))); 18 | map(float, |span: LocatedSpan| { 19 | Token::Float(span.fragment.into(), span.to_span()) 20 | })(input) 21 | } 22 | 23 | pub fn integer(input: LocatedSpan) -> IResult { 24 | map(digit1, |span: LocatedSpan| { 25 | Token::Integer(span.fragment.into(), span.to_span()) 26 | })(input) 27 | } 28 | 29 | #[cfg(test)] 30 | mod tests { 31 | use nom::combinator::all_consuming; 32 | 33 | use super::*; 34 | 35 | fn assert_float_eq(string: &str) { 36 | let span = LocatedSpan::new(string); 37 | match all_consuming(float)(span) { 38 | Ok((_, Token::Float(value, _))) => assert_eq!(value, string), 39 | Ok((_, token)) => panic!("parsing float {:?} produced token: {:?}", string, token), 40 | Err(err) => panic!("parsing float {:?} failed: {:?}", string, err), 41 | } 42 | } 43 | 44 | fn assert_integer_eq(string: &str) { 45 | let span = LocatedSpan::new(string); 46 | match all_consuming(integer)(span) { 47 | Ok((_, Token::Integer(value, _))) => assert_eq!(value, string), 48 | Ok((_, token)) => panic!("parsing integer {:?} produced token: {:?}", string, token), 49 | Err(err) => panic!("parsing integer {:?} failed: {:?}", string, err), 50 | } 51 | } 52 | 53 | #[test] 54 | fn float_literals() { 55 | assert_float_eq("1.23"); 56 | assert_float_eq("6.1E4"); 57 | assert_float_eq("12.0E-3"); 58 | assert_float_eq("6.9E+5"); 59 | assert_float_eq("44.3e5"); 60 | assert_float_eq(".123"); 61 | } 62 | 63 | #[test] 64 | fn integer_literals() { 65 | assert_integer_eq("123"); 66 | assert_integer_eq("00001"); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /nix-parser/src/lexer/lexers/path.rs: -------------------------------------------------------------------------------- 1 | //! Combinators for lexing single paths and path templates. 2 | 3 | use nom::branch::alt; 4 | use nom::bytes::complete::{take_while, take_while1}; 5 | use nom::character::complete::char; 6 | use nom::combinator::{map, opt, recognize}; 7 | use nom::multi::many1; 8 | use nom::sequence::{delimited, pair}; 9 | 10 | use crate::error::Error; 11 | use crate::lexer::util::map_spanned; 12 | use crate::lexer::{IResult, LocatedSpan, Token}; 13 | use crate::ToSpan; 14 | 15 | pub fn path(input: LocatedSpan) -> IResult { 16 | let segments = many1(pair(char('/'), take_while1(path_segment))); 17 | let normal = map(pair(take_while(path_segment), segments), |_| ()); 18 | let segments = many1(pair(char('/'), take_while1(path_segment))); 19 | let home = map(pair(char('~'), segments), |_| ()); 20 | let (remaining, path) = recognize(pair(alt((normal, home)), opt(char('/'))))(input)?; 21 | 22 | if !path.fragment.ends_with('/') { 23 | Ok((remaining, Token::Path(path.fragment.into(), path.to_span()))) 24 | } else { 25 | let message = "paths cannot have trailing slashes".into(); 26 | let error = Error::Message(path.to_span(), message); 27 | let token = Token::Unknown(path.fragment.into(), path.to_span(), error); 28 | Ok((remaining, token)) 29 | } 30 | } 31 | 32 | fn path_segment(c: char) -> bool { 33 | c.is_alphanumeric() || c == '.' || c == '_' || c == '-' || c == '+' 34 | } 35 | 36 | pub fn path_template(input: LocatedSpan) -> IResult { 37 | let name = take_while1(|c: char| c.is_alphanumeric() || "/._-+".contains(c)); 38 | let template = delimited(char('<'), name, char('>')); 39 | map_spanned(template, |span, text| { 40 | Token::PathTemplate(text.fragment.into(), span) 41 | })(input) 42 | } 43 | 44 | #[cfg(test)] 45 | mod tests { 46 | use nom::combinator::all_consuming; 47 | 48 | use super::*; 49 | 50 | fn assert_path_eq(string: &str) { 51 | let span = LocatedSpan::new(string); 52 | match all_consuming(path)(span) { 53 | Ok((_, Token::Path(value, _))) => assert_eq!(value, string), 54 | Ok((_, token)) => panic!("parsing path {:?} produced token: {:?}", string, token), 55 | Err(err) => panic!("parsing path {:?} failed: {:?}", string, err), 56 | } 57 | } 58 | 59 | fn assert_path_template_eq(string: &str, expected: &str) { 60 | let span = LocatedSpan::new(string); 61 | match all_consuming(path_template)(span) { 62 | Ok((_, Token::PathTemplate(value, _))) => assert_eq!(value, expected), 63 | Ok((_, token)) => panic!("parsing template {:?} produced token: {:?}", string, token), 64 | Err(err) => panic!("parsing template {:?} failed: {:?}", string, err), 65 | } 66 | } 67 | 68 | #[test] 69 | fn absolute_paths() { 70 | assert_path_eq("/."); 71 | assert_path_eq("/foo/bar"); 72 | assert_path_eq("/a/b.c/d+e/F_G/h-i/123"); 73 | assert_path_eq("~/foo/bar"); 74 | assert_path_eq("~/a/b.c/d+e/F_G/h-i/123"); 75 | } 76 | 77 | #[test] 78 | fn relative_paths() { 79 | assert_path_eq("./."); 80 | assert_path_eq("../."); 81 | assert_path_eq("./foo/bar"); 82 | assert_path_eq("./a/b.c/d+e/F_G/h-i/123"); 83 | assert_path_eq("foo/bar"); 84 | } 85 | 86 | #[test] 87 | fn path_templates() { 88 | assert_path_template_eq("", "nixpkgs"); 89 | assert_path_template_eq("", "Foo.bar-baz_quux+123"); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /nix-parser/src/lexer/lexers/string.rs: -------------------------------------------------------------------------------- 1 | //! Combinators for lexing single and multi-line strings. 2 | 3 | use codespan::Span; 4 | use nom::branch::alt; 5 | use nom::bytes::complete::{escaped_transform, is_not, tag}; 6 | use nom::character::complete::{anychar, char, multispace0, one_of}; 7 | use nom::combinator::{cond, map, not, peek, recognize}; 8 | use nom::multi::many_till; 9 | use nom::sequence::{pair, terminated}; 10 | 11 | use super::punct_interpolate; 12 | use crate::lexer::util::split_lines_without_indent; 13 | use crate::lexer::{token, IResult, LocatedSpan, StringFragment, StringKind, Token}; 14 | use crate::ToSpan; 15 | 16 | pub fn string(input: LocatedSpan) -> IResult { 17 | let single = string_body(tag("\""), StringKind::Normal, false); 18 | let multi = string_body(tag("''"), StringKind::Indented, true); 19 | alt((single, multi))(input) 20 | } 21 | 22 | fn string_body<'a, F>( 23 | delimiter: F, 24 | kind: StringKind, 25 | is_multiline: bool, 26 | ) -> impl Fn(LocatedSpan<'a>) -> IResult<'a, Token> 27 | where 28 | F: Fn(LocatedSpan<'a>) -> IResult, 29 | { 30 | move |input| { 31 | let start = input; 32 | let (input, _) = pair(&delimiter, cond(is_multiline, multispace0))(input)?; 33 | 34 | let mut remaining = input; 35 | let mut fragments = Vec::new(); 36 | let indent_level = input.get_column(); 37 | 38 | loop { 39 | if let Ok((input, _)) = pair(&delimiter, peek(not(char('$'))))(remaining) { 40 | remaining = input; 41 | break; 42 | } else if let Ok((input, _)) = punct_interpolate(remaining) { 43 | let start = remaining; 44 | remaining = input; 45 | 46 | let mut tokens = Vec::new(); 47 | let mut depth = 1; 48 | while let Ok((input, token)) = token(remaining) { 49 | remaining = input; 50 | match token { 51 | Token::LBrace(_) | Token::Interpolate(_) => depth += 1, 52 | Token::RBrace(_) => { 53 | depth -= 1; 54 | if depth == 0 { 55 | break; 56 | } 57 | } 58 | _ => {} 59 | } 60 | tokens.push(token); 61 | } 62 | 63 | let span = Span::new(start.offset as u32, remaining.offset as u32); 64 | fragments.push(StringFragment::Interpolation(tokens, span)); 65 | } else { 66 | let (string, span) = if is_multiline { 67 | let unescaped_delim = terminated(&delimiter, peek(not(char('$')))); 68 | let boundary = alt((unescaped_delim, recognize(punct_interpolate))); 69 | let chars = alt((tag("''$"), recognize(anychar))); 70 | let (input, string) = recognize(many_till(chars, peek(boundary)))(remaining)?; 71 | let lines: Vec<_> = split_lines_without_indent(string, indent_level).collect(); 72 | remaining = input; 73 | (lines.join("\n").replace("''$", "$"), string.to_span()) 74 | } else { 75 | let unescaped_delim = recognize(terminated(not(char('\\')), peek(&delimiter))); 76 | let boundary = alt((unescaped_delim, peek(recognize(punct_interpolate)))); 77 | let escape = recognize(pair(char('\\'), one_of("\\\"$"))); 78 | let chars = alt((escape, recognize(anychar))); 79 | let (input, string) = recognize(many_till(chars, boundary))(remaining)?; 80 | let (_, escaped) = escaped_transform(is_not("\\"), '\\', escape_codes)(string)?; 81 | remaining = input; 82 | (escaped, string.to_span()) 83 | }; 84 | 85 | fragments.push(StringFragment::Literal(string, span)); 86 | } 87 | } 88 | 89 | let span = Span::new(start.offset as u32, remaining.offset as u32); 90 | Ok((remaining, Token::String(fragments, kind, span))) 91 | } 92 | } 93 | 94 | fn escape_codes(input: LocatedSpan) -> IResult { 95 | alt(( 96 | map(char('n'), |_| '\n'), 97 | map(char('r'), |_| '\r'), 98 | map(char('t'), |_| '\t'), 99 | anychar, 100 | ))(input) 101 | } 102 | -------------------------------------------------------------------------------- /nix-parser/src/lexer/lexers/uri.rs: -------------------------------------------------------------------------------- 1 | //! Combinator for lexing URIs. 2 | 3 | use nom::bytes::complete::{take_while, take_while1}; 4 | use nom::character::complete::{anychar, char}; 5 | use nom::combinator::{recognize, verify}; 6 | use nom::sequence::{pair, tuple}; 7 | 8 | use crate::lexer::util::map_spanned; 9 | use crate::lexer::{IResult, LocatedSpan, Token}; 10 | 11 | pub fn uri(input: LocatedSpan) -> IResult { 12 | let first = verify(anychar, |c| c.is_alphabetic()); 13 | let rest = take_while(|c: char| c.is_alphanumeric() || "+-.".contains(c)); 14 | let scheme = pair(first, rest); 15 | 16 | let path = take_while1(|c: char| c.is_alphanumeric() || "%/?:@&=+$,-_.!~*'".contains(c)); 17 | let uri = recognize(tuple((scheme, char(':'), path))); 18 | 19 | map_spanned(uri, |span, uri| Token::Uri(uri.fragment.into(), span))(input) 20 | } 21 | 22 | #[cfg(test)] 23 | mod tests { 24 | use nom::combinator::all_consuming; 25 | 26 | use super::*; 27 | 28 | fn assert_uri_eq(string: &str) { 29 | let span = LocatedSpan::new(string); 30 | match all_consuming(uri)(span) { 31 | Ok((_, Token::Uri(value, _))) => assert_eq!(value, string), 32 | Ok((_, token)) => panic!("parsing {:?} produced token: {:?}", string, token), 33 | Err(err) => panic!("parsing {:?} failed: {:?}", string, err), 34 | } 35 | } 36 | 37 | #[test] 38 | fn short_uri() { 39 | assert_uri_eq("x:x"); 40 | } 41 | 42 | #[test] 43 | fn with_explicit_port() { 44 | assert_uri_eq("https://svn.cs.uu.nl:12443/repos/trace/trunk"); 45 | } 46 | 47 | #[test] 48 | fn font_pack_uri() { 49 | assert_uri_eq( 50 | "http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2", 51 | ); 52 | } 53 | 54 | #[test] 55 | fn academic_uri() { 56 | assert_uri_eq("http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz"); 57 | } 58 | 59 | #[test] 60 | fn complex_uri() { 61 | assert_uri_eq("http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz"); 62 | } 63 | 64 | #[test] 65 | fn ftp_uri() { 66 | assert_uri_eq("ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz"); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /nix-parser/src/lexer/util.rs: -------------------------------------------------------------------------------- 1 | //! Utility functions for the lexer. 2 | 3 | use codespan::Span; 4 | use nom::Slice; 5 | 6 | use super::{IResult, LocatedSpan, Token}; 7 | use crate::error::{Errors, IncorrectDelimError, UnclosedDelimError}; 8 | use crate::ToSpan; 9 | 10 | /// Combinator which behaves like `nom::combinator::map()`, except it also includes a `Span` based 11 | /// on the consumed input. 12 | pub fn map_spanned<'a, O1, O2, P, F>(parser: P, f: F) -> impl Fn(LocatedSpan<'a>) -> IResult 13 | where 14 | P: Fn(LocatedSpan<'a>) -> IResult, 15 | F: Fn(Span, O1) -> O2, 16 | { 17 | move |input| { 18 | let (remainder, value) = parser(input)?; 19 | let value_len = remainder.offset - input.offset; 20 | let span = input.slice(..value_len).to_span(); 21 | Ok((remainder, f(span, value))) 22 | } 23 | } 24 | 25 | /// Checks whether all delimiter tokens (braces, brackets, and parentheses) in the given token 26 | /// slice are balanced. 27 | /// 28 | /// If all delimiters are balanced, the returned `Errors` stack will be empty. 29 | pub fn check_delims_balanced(tokens: &[Token], eof_span: Span) -> Errors { 30 | let mut delim_stack = Vec::new(); 31 | let mut errors = Errors::new(); 32 | 33 | for token in tokens.iter() { 34 | match token { 35 | delim @ Token::Interpolate(_) => delim_stack.push(delim), 36 | delim @ Token::LBrace(_) => delim_stack.push(delim), 37 | delim @ Token::LBracket(_) => delim_stack.push(delim), 38 | delim @ Token::LParen(_) => delim_stack.push(delim), 39 | Token::RBrace(span) => match delim_stack.pop() { 40 | None | Some(Token::LBrace(_)) | Some(Token::Interpolate(_)) => {} 41 | Some(token) => { 42 | let unclosed = token.to_span(); 43 | errors.push(IncorrectDelimError::new('}', *span, None, Some(unclosed))); 44 | } 45 | }, 46 | Token::RBracket(span) => match delim_stack.pop() { 47 | None | Some(Token::LBracket(_)) => {} 48 | Some(token) => { 49 | let unclosed = token.to_span(); 50 | errors.push(IncorrectDelimError::new(']', *span, None, Some(unclosed))); 51 | } 52 | }, 53 | Token::RParen(span) => match delim_stack.pop() { 54 | None | Some(Token::LParen(_)) => {} 55 | Some(token) => { 56 | let unclosed = token.to_span(); 57 | errors.push(IncorrectDelimError::new(')', *span, None, Some(unclosed))); 58 | } 59 | }, 60 | _ => continue, 61 | } 62 | } 63 | 64 | if !delim_stack.is_empty() { 65 | errors.push(UnclosedDelimError::new(delim_stack, eof_span)); 66 | } 67 | 68 | errors 69 | } 70 | 71 | /// Splits the given input string into lines with the leading indentation trimmed on all subsequent 72 | /// lines. 73 | /// 74 | /// # Panics 75 | /// 76 | /// This function will panic if `indent` not greater than 0. Since this parameter is expected to be 77 | /// populated with a value from `LocatedSpan::get_column()`, it is expected that the indentation 78 | /// column is always 1 or more. 79 | pub fn split_lines_without_indent(input: LocatedSpan, indent: usize) -> impl Iterator { 80 | debug_assert!(indent > 0); 81 | input.fragment.split('\n').enumerate().map(move |(i, row)| { 82 | if i > 0 { 83 | let trim_start = row 84 | .char_indices() 85 | .take_while(|(i, c)| c.is_whitespace() && *i < indent - 1) 86 | .count(); 87 | &row[trim_start..] 88 | } else { 89 | row 90 | } 91 | }) 92 | } 93 | 94 | #[cfg(test)] 95 | mod tests { 96 | use super::*; 97 | 98 | #[test] 99 | fn split_lines() { 100 | let input = LocatedSpan::new( 101 | r#" 102 | first 103 | second 104 | indented 105 | de-indented"#, 106 | ); 107 | let lines: Vec<_> = split_lines_without_indent(input, 13).collect(); 108 | assert_eq!(lines, ["", "first", "second", " indented", "de-indented"]); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /nix-parser/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Parser for the Nix programming language. 2 | 3 | #![forbid(unsafe_code)] 4 | #![recursion_limit = "128"] 5 | 6 | use codespan::Span; 7 | 8 | pub mod ast; 9 | pub mod error; 10 | pub mod lexer; 11 | pub mod parser; 12 | 13 | pub trait HasSpan { 14 | fn span(&self) -> Span; 15 | } 16 | 17 | impl HasSpan for Span { 18 | fn span(&self) -> Span { 19 | *self 20 | } 21 | } 22 | 23 | pub trait ToSpan { 24 | fn to_span(&self) -> Span; 25 | } 26 | 27 | impl ToSpan for Span { 28 | fn to_span(&self) -> Span { 29 | *self 30 | } 31 | } 32 | 33 | impl<'a, T: ToSpan> ToSpan for &'a T { 34 | fn to_span(&self) -> Span { 35 | (*self).to_span() 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /nix-parser/src/parser.rs: -------------------------------------------------------------------------------- 1 | //! Functions for parsing expressions. 2 | 3 | pub use self::partial::Partial; 4 | 5 | use std::str::FromStr; 6 | 7 | use nom::combinator::{all_consuming, map, opt}; 8 | use nom::sequence::terminated; 9 | 10 | use self::partial::{map_partial, pair_partial}; 11 | use crate::ast::{Expr, SourceFile}; 12 | use crate::error::Errors; 13 | use crate::lexer::{Lexer, Tokens}; 14 | 15 | mod expr; 16 | mod partial; 17 | mod tokens; 18 | 19 | type IResult<'a, T> = nom::IResult, T, Errors>; 20 | 21 | impl FromStr for Expr { 22 | type Err = Errors; 23 | 24 | fn from_str(s: &str) -> Result { 25 | parse_expr(s) 26 | } 27 | } 28 | 29 | impl FromStr for SourceFile { 30 | type Err = Errors; 31 | 32 | fn from_str(s: &str) -> Result { 33 | parse_source_file(s) 34 | } 35 | } 36 | 37 | pub fn parse_expr(expr: &str) -> Result { 38 | parse_expr_partial(expr).and_then(|partial| partial.verify()) 39 | } 40 | 41 | pub fn parse_expr_partial(expr: &str) -> Result, Errors> { 42 | let lexer = Lexer::new(expr)?; 43 | let tokens = lexer.tokens(); 44 | let errors = lexer.errors().clone(); 45 | 46 | let mut partial = match all_consuming(terminated(expr::expr, tokens::eof))(tokens) { 47 | Ok((_, partial)) => partial, 48 | Err(nom::Err::Incomplete(_)) => panic!("file was incomplete"), 49 | Err(nom::Err::Error(mut err)) | Err(nom::Err::Failure(mut err)) => { 50 | err.extend(errors); 51 | return Err(err); 52 | } 53 | }; 54 | 55 | partial.extend_errors(errors); 56 | Ok(partial) 57 | } 58 | 59 | pub fn parse_source_file(source: &str) -> Result { 60 | parse_source_file_partial(source).and_then(|partial| partial.verify()) 61 | } 62 | 63 | pub fn parse_source_file_partial(source: &str) -> Result, Errors> { 64 | let lexer = Lexer::new(source)?; 65 | let tokens = lexer.tokens(); 66 | let errors = lexer.errors().clone(); 67 | 68 | let expr = pair_partial(map(opt(tokens::comment), Partial::from), expr::expr); 69 | let source_file = map_partial(expr, |(comment, expr)| SourceFile::new(comment, expr)); 70 | let mut partial = match all_consuming(terminated(source_file, tokens::eof))(tokens) { 71 | Ok((_, partial)) => partial, 72 | Err(nom::Err::Incomplete(_)) => panic!("file was incomplete"), 73 | Err(nom::Err::Error(mut err)) | Err(nom::Err::Failure(mut err)) => { 74 | err.extend(errors); 75 | return Err(err); 76 | } 77 | }; 78 | 79 | partial.extend_errors(errors); 80 | Ok(partial) 81 | } 82 | 83 | #[cfg(test)] 84 | mod tests { 85 | use super::*; 86 | 87 | use crate::nix_expr_and_str; 88 | 89 | #[test] 90 | fn issue_6() { 91 | let (expected, s) = nix_expr_and_str!({ 92 | baz = quux; 93 | inherit ({foo}: foo) bar; 94 | }); 95 | let found = parse_expr(&s).unwrap(); 96 | assert_eq!(expected, found); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /nix-parser/src/parser/expr.rs: -------------------------------------------------------------------------------- 1 | use std::iter::{self, FromIterator}; 2 | 3 | use codespan::Span; 4 | use nom::branch::alt; 5 | use nom::bytes::complete::take; 6 | use nom::combinator::{map, opt}; 7 | use nom::multi::{many0, many0_count}; 8 | use nom::sequence::{pair, preceded}; 9 | 10 | use super::partial::{ 11 | expect_terminated, map_partial, map_partial_spanned, opt_partial, pair_partial, Partial, 12 | }; 13 | use super::{tokens, IResult}; 14 | use crate::ast::tokens::Ident; 15 | use crate::ast::{BinaryOp, Expr, ExprBinary, ExprFnApp, ExprIf, ExprProj, ExprUnary, UnaryOp}; 16 | use crate::error::{Error, Errors, UnexpectedError}; 17 | use crate::lexer::{Token, Tokens}; 18 | use crate::{HasSpan, ToSpan}; 19 | 20 | mod atomic; 21 | mod attr; 22 | mod bind; 23 | mod func; 24 | mod stmt; 25 | mod util; 26 | 27 | pub fn expr(input: Tokens) -> IResult> { 28 | preceded(many0_count(tokens::comment), function)(input) 29 | } 30 | 31 | fn function(input: Tokens) -> IResult> { 32 | let function = map_partial(func::fn_decl, Expr::from); 33 | let with = map_partial(stmt::with, Expr::from); 34 | let assert = map_partial(stmt::assert, Expr::from); 35 | let let_in = map_partial(stmt::let_in, Expr::from); 36 | alt((function, with, assert, let_in, if_else))(input) 37 | } 38 | 39 | fn if_else(input: Tokens) -> IResult> { 40 | let cond = alt((util::error_expr_if(tokens::keyword_then), expr)); 41 | let cond_then = expect_terminated(cond, tokens::keyword_then); 42 | let if_cond_then = preceded(tokens::keyword_if, cond_then); 43 | 44 | let body = alt((util::error_expr_if(tokens::keyword_else), expr)); 45 | let body_else = expect_terminated(body, tokens::keyword_else); 46 | 47 | let expr = alt((util::error_expr_if(tokens::eof), expr)); 48 | let block = pair_partial(if_cond_then, pair_partial(body_else, expr)); 49 | let if_else = map_partial_spanned(block, |span, (cond, (body, fallback))| { 50 | Expr::from(ExprIf::new(cond, body, fallback, span)) 51 | }); 52 | 53 | alt((if_else, imply))(input) 54 | } 55 | 56 | fn imply(input: Tokens) -> IResult> { 57 | let (input, first) = and(input)?; 58 | let next = alt((and, util::error_expr_if(tokens::eof))); 59 | util::fold_many0(preceded(tokens::op_imply, next), first, |lhs, rhs| { 60 | lhs.flat_map(|lhs| { 61 | rhs.map(|rhs| { 62 | let span = Span::merge(lhs.span(), rhs.span()); 63 | Expr::from(ExprBinary::new(BinaryOp::Impl, lhs, rhs, span)) 64 | }) 65 | }) 66 | })(input) 67 | } 68 | 69 | fn and(input: Tokens) -> IResult> { 70 | let (input, first) = or(input)?; 71 | let next = alt((or, util::error_expr_if(tokens::eof))); 72 | util::fold_many0(preceded(tokens::op_and, next), first, |lhs, rhs| { 73 | lhs.flat_map(|lhs| { 74 | rhs.map(|rhs| { 75 | let span = Span::merge(lhs.span(), rhs.span()); 76 | Expr::from(ExprBinary::new(BinaryOp::And, lhs, rhs, span)) 77 | }) 78 | }) 79 | })(input) 80 | } 81 | 82 | fn or(input: Tokens) -> IResult> { 83 | let (input, first) = equality(input)?; 84 | let next = alt((equality, util::error_expr_if(tokens::eof))); 85 | util::fold_many0(preceded(tokens::op_or, next), first, |lhs, rhs| { 86 | lhs.flat_map(|lhs| { 87 | rhs.map(|rhs| { 88 | let span = Span::merge(lhs.span(), rhs.span()); 89 | Expr::from(ExprBinary::new(BinaryOp::Or, lhs, rhs, span)) 90 | }) 91 | }) 92 | })(input) 93 | } 94 | 95 | fn equality(input: Tokens) -> IResult> { 96 | let eq = map(tokens::op_eq, |_| BinaryOp::Eq); 97 | let neq = map(tokens::op_neq, |_| BinaryOp::NotEq); 98 | let rhs = alt((compare, util::error_expr_if(tokens::eof))); 99 | let expr = pair(compare, opt(pair(alt((eq, neq)), rhs))); 100 | map(expr, |(lhs, op)| match op { 101 | None => lhs, 102 | Some((op, rhs)) => lhs.flat_map(|lhs| { 103 | rhs.map(|rhs| { 104 | let span = Span::merge(lhs.span(), rhs.span()); 105 | Expr::from(ExprBinary::new(op, lhs, rhs, span)) 106 | }) 107 | }), 108 | })(input) 109 | } 110 | 111 | fn compare(input: Tokens) -> IResult> { 112 | let lte = map(tokens::op_lte, |_| BinaryOp::LessThanEq); 113 | let lt = map(tokens::op_lt, |_| BinaryOp::LessThan); 114 | let gte = map(tokens::op_gte, |_| BinaryOp::GreaterThanEq); 115 | let gt = map(tokens::op_gt, |_| BinaryOp::GreaterThan); 116 | 117 | let rhs = alt((update, util::error_expr_if(tokens::eof))); 118 | let expr = pair(update, opt(pair(alt((lte, lt, gte, gt)), rhs))); 119 | map(expr, |(lhs, op)| match op { 120 | None => lhs, 121 | Some((op, rhs)) => lhs.flat_map(|lhs| { 122 | rhs.map(|rhs| { 123 | let span = Span::merge(lhs.span(), rhs.span()); 124 | Expr::from(ExprBinary::new(op, lhs, rhs, span)) 125 | }) 126 | }), 127 | })(input) 128 | } 129 | 130 | fn update(input: Tokens) -> IResult> { 131 | let rhs = alt((sum, util::error_expr_if(tokens::eof))); 132 | let expr = pair(sum, many0(preceded(tokens::op_update, rhs))); 133 | map(expr, |(first, rest)| { 134 | if rest.is_empty() { 135 | first 136 | } else { 137 | let exprs = Partial::from_iter(iter::once(first).chain(rest)); 138 | exprs.map(|mut exprs| { 139 | let last = exprs.pop().unwrap(); 140 | exprs.into_iter().rev().fold(last, |rhs, lhs| { 141 | let span = Span::merge(lhs.span(), rhs.span()); 142 | Expr::from(ExprBinary::new(BinaryOp::Update, lhs, rhs, span)) 143 | }) 144 | }) 145 | } 146 | })(input) 147 | } 148 | 149 | fn sum(input: Tokens) -> IResult> { 150 | let (input, first) = product(input)?; 151 | let next = alt((product, util::error_expr_if(tokens::eof))); 152 | let add = map(tokens::op_add, |_| BinaryOp::Add); 153 | let sub = map(tokens::op_sub, |_| BinaryOp::Sub); 154 | util::fold_many0(pair(alt((add, sub)), next), first, |lhs, (op, rhs)| { 155 | lhs.flat_map(|lhs| { 156 | rhs.map(|rhs| { 157 | let span = Span::merge(lhs.span(), rhs.span()); 158 | Expr::from(ExprBinary::new(op, lhs, rhs, span)) 159 | }) 160 | }) 161 | })(input) 162 | } 163 | 164 | fn product(input: Tokens) -> IResult> { 165 | let (input, first) = concat(input)?; 166 | let next = alt((concat, util::error_expr_if(tokens::eof))); 167 | let mul = map(tokens::op_mul, |_| BinaryOp::Mul); 168 | let div = map(tokens::op_div, |_| BinaryOp::Div); 169 | util::fold_many0(pair(alt((mul, div)), next), first, |lhs, (op, rhs)| { 170 | lhs.flat_map(|lhs| { 171 | rhs.map(|rhs| { 172 | let span = Span::merge(lhs.span(), rhs.span()); 173 | Expr::from(ExprBinary::new(op, lhs, rhs, span)) 174 | }) 175 | }) 176 | })(input) 177 | } 178 | 179 | fn concat(input: Tokens) -> IResult> { 180 | let expr = pair(has_attr, many0(preceded(tokens::op_concat, has_attr))); 181 | map(expr, |(first, rest)| { 182 | if rest.is_empty() { 183 | first 184 | } else { 185 | let exprs = Partial::from_iter(iter::once(first).chain(rest)); 186 | exprs.map(|mut exprs| { 187 | let last = exprs.pop().unwrap(); 188 | exprs.into_iter().rev().fold(last, |rhs, lhs| { 189 | let span = Span::merge(lhs.span(), rhs.span()); 190 | Expr::from(ExprBinary::new(BinaryOp::Concat, lhs, rhs, span)) 191 | }) 192 | }) 193 | } 194 | })(input) 195 | } 196 | 197 | fn has_attr(input: Tokens) -> IResult> { 198 | let rhs = alt((unary, util::error_expr_if(tokens::eof))); 199 | let expr = pair(unary, opt(preceded(tokens::op_question, rhs))); 200 | map(expr, |(lhs, rhs)| match rhs { 201 | None => lhs, 202 | Some(rhs) => lhs.flat_map(|lhs| { 203 | rhs.map(|rhs| { 204 | let span = Span::merge(lhs.span(), rhs.span()); 205 | Expr::from(ExprBinary::new(BinaryOp::HasAttr, lhs, rhs, span)) 206 | }) 207 | }), 208 | })(input) 209 | } 210 | 211 | fn unary(input: Tokens) -> IResult> { 212 | let neg = map(tokens::op_sub, |_| UnaryOp::Neg); 213 | let not = map(tokens::op_not, |_| UnaryOp::Not); 214 | let unary = pair_partial(map(opt(alt((neg, not))), Partial::from), fn_app); 215 | let expr = map_partial_spanned(unary, |span, (unary, expr)| match unary { 216 | Some(op) => Expr::from(ExprUnary::new(op, expr, span)), 217 | None => expr, 218 | }); 219 | alt((expr, error))(input) 220 | } 221 | 222 | fn fn_app(input: Tokens) -> IResult> { 223 | let (input, first) = project(input)?; 224 | util::fold_many0(project, first, |lhs, rhs| { 225 | lhs.flat_map(|lhs| { 226 | rhs.map(|rhs| { 227 | let span = Span::merge(lhs.span(), rhs.span()); 228 | Expr::from(ExprFnApp::new(lhs, rhs, span)) 229 | }) 230 | }) 231 | })(input) 232 | } 233 | 234 | fn project(input: Tokens) -> IResult> { 235 | let (input, mut atomic) = atomic(input)?; 236 | 237 | if let Ok((remaining, dot)) = tokens::dot(input) { 238 | let default = alt((project, error, util::error_expr_if(tokens::eof))); 239 | let or_default = opt_partial(preceded(tokens::keyword_or, default)); 240 | 241 | if let Ok((remaining, path)) = pair_partial(attr::attr_path, or_default)(remaining) { 242 | let proj = atomic.flat_map(|atomic| { 243 | path.map(|(path, default)| { 244 | let span = Span::merge(atomic.span(), path.span()); 245 | match default { 246 | Some(expr) => Expr::from(ExprProj::new(atomic, path, Some(expr), span)), 247 | None => Expr::from(ExprProj::new(atomic, path, None, span)), 248 | } 249 | }) 250 | }); 251 | 252 | Ok((remaining, proj)) 253 | } else { 254 | let error = UnexpectedError::new(input.current().description(), dot); 255 | atomic.extend_errors(iter::once(error.into())); 256 | Ok((remaining, atomic)) 257 | } 258 | } else if let Ok((remaining, or_span)) = tokens::keyword_or(input) { 259 | let expr = atomic.map(|atomic| { 260 | let arg = Expr::Ident(Ident::from(("or", or_span))); 261 | let span = Span::merge(atomic.span(), or_span); 262 | Expr::from(ExprFnApp::new(atomic, arg, span)) 263 | }); 264 | 265 | Ok((remaining, expr)) 266 | } else { 267 | Ok((input, atomic)) 268 | } 269 | } 270 | 271 | fn atomic(input: Tokens) -> IResult> { 272 | let ident = map_partial(atomic::identifier, Expr::from); 273 | let string = map_partial(atomic::string, Expr::from); 274 | let literal = map_partial(atomic::literal, Expr::from); 275 | let paren = map_partial(atomic::paren, Expr::from); 276 | let set = map_partial(atomic::set, Expr::from); 277 | let list = map_partial(atomic::list, Expr::from); 278 | let rec_set = map_partial(atomic::rec_set, Expr::from); 279 | let let_set = map_partial(atomic::let_set, Expr::from); 280 | alt((ident, string, literal, paren, set, list, rec_set, let_set))(input) 281 | } 282 | 283 | fn error(input: Tokens) -> IResult> { 284 | let (remaining, tokens) = take(1usize)(input)?; 285 | 286 | let mut errors = Errors::new(); 287 | let expr = match tokens.current() { 288 | Token::Eof(_) => return Err(nom::Err::Error(errors)), 289 | Token::Interpolation(.., span) => { 290 | let message = "interpolation not permitted in this position".into(); 291 | errors.push(Error::Message(*span, message)); 292 | Expr::Error(*span) 293 | } 294 | token => { 295 | errors.push(UnexpectedError::new(token.description(), token.to_span())); 296 | Expr::Error(token.to_span()) 297 | } 298 | }; 299 | 300 | Ok((remaining, Partial::with_errors(Some(expr), errors))) 301 | } 302 | -------------------------------------------------------------------------------- /nix-parser/src/parser/expr/atomic.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use nom::branch::alt; 4 | use nom::bytes::complete::take; 5 | use nom::combinator::map; 6 | use nom::error::ErrorKind; 7 | use nom::multi::{many0, many0_count}; 8 | use nom::sequence::{pair, preceded, terminated}; 9 | 10 | use super::{bind, error, expr, project}; 11 | use crate::ast::tokens::{Ident, Literal}; 12 | use crate::ast::{ 13 | Bind, Expr, ExprInterpolation, ExprLet, ExprList, ExprParen, ExprRec, ExprSet, ExprString, 14 | StringFragment, 15 | }; 16 | use crate::error::{Error, Errors}; 17 | use crate::lexer::{StringFragment as LexerFragment, Token, Tokens}; 18 | use crate::parser::partial::{expect_terminated, many_till_partial, map_partial_spanned, Partial}; 19 | use crate::parser::{tokens, IResult}; 20 | use crate::ToSpan; 21 | 22 | pub fn paren(input: Tokens) -> IResult> { 23 | let expr = terminated(expr, many0_count(tokens::comment)); 24 | let paren = expect_terminated(preceded(tokens::paren_left, expr), tokens::paren_right); 25 | map_partial_spanned(paren, |span, inner| ExprParen::new(inner, span))(input) 26 | } 27 | 28 | pub fn set(input: Tokens) -> IResult> { 29 | map_partial_spanned(set_binds, |span, binds| ExprSet::new(binds, span))(input) 30 | } 31 | 32 | pub fn rec_set(input: Tokens) -> IResult> { 33 | let rec_set = preceded(tokens::keyword_rec, set_binds); 34 | map_partial_spanned(rec_set, |span, binds| ExprRec::new(binds, span))(input) 35 | } 36 | 37 | pub fn let_set(input: Tokens) -> IResult> { 38 | let let_set = preceded(tokens::keyword_let, set_binds); 39 | map_partial_spanned(let_set, |span, binds| ExprLet::new(binds, span))(input) 40 | } 41 | 42 | fn set_binds(input: Tokens) -> IResult>> { 43 | let term = alt((tokens::brace_right, tokens::semi)); 44 | let binds = many_till_partial(bind::bind, pair(many0_count(tokens::comment), term)); 45 | let set = terminated(binds, many0_count(tokens::comment)); 46 | expect_terminated(preceded(tokens::brace_left, set), tokens::brace_right)(input) 47 | } 48 | 49 | pub fn list(input: Tokens) -> IResult> { 50 | let elems = many_till_partial(list_elem, tokens::bracket_right); 51 | let inner = preceded(many0(tokens::comment), elems); 52 | let list = expect_terminated(preceded(tokens::bracket_left, inner), tokens::bracket_right); 53 | map_partial_spanned(list, |span, exprs| ExprList::new(exprs, span))(input) 54 | } 55 | 56 | fn list_elem(input: Tokens) -> IResult> { 57 | let mut input = input; 58 | let mut errors = Errors::new(); 59 | 60 | let (remaining, tokens) = take(1usize)(input)?; 61 | match tokens.current() { 62 | Token::Sub(span) => { 63 | let message = "unary negation `-` not allowed in lists".into(); 64 | errors.push(Error::Message(*span, message)); 65 | input = remaining; 66 | } 67 | Token::Not(span) => { 68 | let message = "logical not `!` not allowed in lists".into(); 69 | errors.push(Error::Message(*span, message)); 70 | input = remaining; 71 | } 72 | _ => {} 73 | } 74 | 75 | let (_, tokens) = take(1usize)(input)?; 76 | if let Token::RBracket(_) = tokens.current() { 77 | return Err(nom::Err::Error(errors)); 78 | } 79 | 80 | let element = alt((project, error)); 81 | let (remaining, mut proj) = terminated(element, many0_count(tokens::comment))(input)?; 82 | proj.extend_errors(errors); 83 | Ok((remaining, proj)) 84 | } 85 | 86 | pub fn string(input: Tokens) -> IResult> { 87 | let (remaining, (fragments, _kind, span)) = tokens::string(input)?; 88 | let mut parts = Vec::with_capacity(fragments.len()); 89 | 90 | for frag in fragments { 91 | match frag { 92 | LexerFragment::Literal(text, span) => { 93 | parts.push(Partial::from(StringFragment::Literal(text.clone(), *span))); 94 | } 95 | LexerFragment::Interpolation(tokens, span) => { 96 | let expr = if tokens.is_empty() { 97 | let mut errors = Errors::new(); 98 | let message = "interpolation cannot be empty".into(); 99 | errors.push(Error::Message(*span, message)); 100 | Partial::with_errors(Some(Expr::Error(*span)), errors) 101 | } else { 102 | let (_, expr) = expr(Tokens::new(&tokens))?; 103 | expr 104 | }; 105 | 106 | parts.push(expr.map(|expr| { 107 | StringFragment::Interpolation(ExprInterpolation::new(expr, *span)) 108 | })); 109 | } 110 | } 111 | } 112 | 113 | let partial: Partial> = parts.into_iter().collect(); 114 | Ok((remaining, partial.map(|frags| ExprString::new(frags, span)))) 115 | } 116 | 117 | pub fn literal(input: Tokens) -> IResult> { 118 | let (remaining, tokens) = take(1usize)(input)?; 119 | let literal = match tokens.current() { 120 | Token::Boolean(value, span) => Literal::Boolean(*value, *span), 121 | Token::Null(span) => Literal::Null(*span), 122 | Token::Path(value, span) => Literal::Path(PathBuf::from(value.to_string()), *span), 123 | Token::Float(value, span) => { 124 | let float: f64 = lexical_core::parse(value.as_bytes()).expect("float parsing failed"); 125 | Literal::Float(float, *span) 126 | } 127 | Token::Integer(value, span) => { 128 | Literal::Integer(value.parse().expect("integer parsing failed"), *span) 129 | } 130 | Token::PathTemplate(value, span) => { 131 | Literal::PathTemplate(PathBuf::from(value.to_string()), *span) 132 | } 133 | Token::Uri(value, span) => Literal::Uri(value.parse().expect("URI parsing failed"), *span), 134 | token => { 135 | let mut errors = Errors::new(); 136 | errors.push(Error::Nom(token.to_span(), ErrorKind::Tag)); 137 | return Err(nom::Err::Error(errors)); 138 | } 139 | }; 140 | 141 | Ok((remaining, Partial::from(literal))) 142 | } 143 | 144 | pub fn identifier(input: Tokens) -> IResult> { 145 | map(tokens::identifier, Partial::from)(input) 146 | } 147 | -------------------------------------------------------------------------------- /nix-parser/src/parser/expr/attr.rs: -------------------------------------------------------------------------------- 1 | use std::iter::FromIterator; 2 | 3 | use nom::branch::alt; 4 | use nom::combinator::map; 5 | use nom::multi::separated_nonempty_list; 6 | 7 | use super::{atomic, expr}; 8 | use crate::ast::{AttrPath, AttrSegment, Expr, ExprInterpolation}; 9 | use crate::error::{Error, Errors}; 10 | use crate::lexer::Tokens; 11 | use crate::parser::partial::{map_partial, Partial}; 12 | use crate::parser::{tokens, IResult}; 13 | 14 | pub fn attr_path(input: Tokens) -> IResult> { 15 | let path = separated_nonempty_list(tokens::dot, segment); 16 | map_partial(map(path, Partial::from_iter), AttrPath::new)(input) 17 | } 18 | 19 | fn segment(input: Tokens) -> IResult> { 20 | let identifier = map_partial(atomic::identifier, AttrSegment::Ident); 21 | let string = map_partial(atomic::string, AttrSegment::String); 22 | let interpolation = map_partial(interpolation, AttrSegment::Interpolation); 23 | alt((identifier, string, interpolation))(input) 24 | } 25 | 26 | fn interpolation(input: Tokens) -> IResult> { 27 | let (remaining, (tokens, span)) = tokens::interpolation(input)?; 28 | let expr = if tokens.is_empty() { 29 | let mut errors = Errors::new(); 30 | errors.push(Error::Message(span, "interpolation cannot be empty".into())); 31 | Partial::with_errors(Some(Expr::Error(span)), errors) 32 | } else { 33 | let (_, expr) = expr(Tokens::new(&tokens))?; 34 | expr 35 | }; 36 | 37 | Ok((remaining, expr.map(|e| ExprInterpolation::new(e, span)))) 38 | } 39 | -------------------------------------------------------------------------------- /nix-parser/src/parser/expr/bind.rs: -------------------------------------------------------------------------------- 1 | use codespan::Span; 2 | use nom::branch::alt; 3 | use nom::combinator::map; 4 | use nom::multi::{many0, many0_count}; 5 | use nom::sequence::{pair, preceded}; 6 | 7 | use super::{attr, expr, util}; 8 | use crate::ast::tokens::{Comment, Ident}; 9 | use crate::ast::{Bind, BindInherit, BindInheritExpr, BindSimple}; 10 | use crate::error::{Error, Errors, UnexpectedError}; 11 | use crate::lexer::Tokens; 12 | use crate::parser::partial::{ 13 | expect_terminated, map_partial, map_partial_spanned, pair_partial, Partial, 14 | }; 15 | use crate::parser::{tokens, IResult}; 16 | use crate::{HasSpan, ToSpan}; 17 | 18 | pub fn bind(input: Tokens) -> IResult> { 19 | let inherit_expr = map_partial(inherit_expr, Bind::InheritExpr); 20 | let inherit = map_partial(inherit, Bind::Inherit); 21 | let simple = map_partial(simple, Bind::Simple); 22 | match expect_terminated(alt((inherit_expr, inherit, simple)), tokens::semi)(input) { 23 | Ok(output) => Ok(output), 24 | Err(_) => { 25 | let mut errors = Errors::new(); 26 | let description = input.current().description(); 27 | let span = input.current().to_span(); 28 | errors.push(UnexpectedError::new(description, span)); 29 | Err(nom::Err::Error(errors)) 30 | } 31 | } 32 | } 33 | 34 | fn simple(input: Tokens) -> IResult> { 35 | let error = util::error_expr_if(alt((tokens::semi, tokens::brace_right))); 36 | let lhs = expect_terminated(attr::attr_path, tokens::eq); 37 | let bind = pair_partial(final_comment, pair_partial(lhs, alt((expr, error)))); 38 | map_partial(bind, move |(comment, (attr, expr))| { 39 | let span = Span::merge(attr.span(), expr.span()); 40 | BindSimple::new(comment, attr, expr, span) 41 | })(input) 42 | } 43 | 44 | fn inherit(input: Tokens) -> IResult> { 45 | let keyword_inherit = pair(many0_count(tokens::comment), tokens::keyword_inherit); 46 | let bind = preceded(keyword_inherit, ident_sequence); 47 | map_partial_spanned(bind, |span, idents| BindInherit::new(idents, span))(input) 48 | } 49 | 50 | fn inherit_expr(input: Tokens) -> IResult> { 51 | let keyword_inherit = pair(many0_count(tokens::comment), tokens::keyword_inherit); 52 | let inner = alt((expr, util::error_expr_if(tokens::paren_right))); 53 | let expr = expect_terminated(preceded(tokens::paren_left, inner), tokens::paren_right); 54 | let bind = preceded(keyword_inherit, pair_partial(expr, ident_sequence)); 55 | map_partial_spanned(bind, |span, (expr, idents)| { 56 | BindInheritExpr::new(expr, idents, span) 57 | })(input) 58 | } 59 | 60 | fn final_comment(input: Tokens) -> IResult>> { 61 | map(many0(tokens::comment), |mut comments| { 62 | Partial::new(Some(comments.pop())) 63 | })(input) 64 | } 65 | 66 | fn ident_sequence(input: Tokens) -> IResult>> { 67 | let (remaining, idents) = many0(tokens::identifier)(input)?; 68 | if idents.is_empty() { 69 | let mut errors = Errors::new(); 70 | let span = input.current().to_span(); 71 | let message = "expected at least one identifier".into(); 72 | errors.push(Error::Message(span, message)); 73 | Ok((remaining, Partial::with_errors(None, errors))) 74 | } else { 75 | Ok((remaining, Partial::from(idents))) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /nix-parser/src/parser/expr/func.rs: -------------------------------------------------------------------------------- 1 | use codespan::Span; 2 | use nom::branch::alt; 3 | use nom::combinator::{map, opt}; 4 | use nom::multi::many0_count; 5 | use nom::sequence::{delimited, pair, preceded, terminated}; 6 | 7 | use super::{expr, util}; 8 | use crate::ast::{ExprFnDecl, Formal, Pattern, SetPattern}; 9 | use crate::lexer::Tokens; 10 | use crate::parser::partial::{ 11 | map_partial, map_partial_spanned, pair_partial, separated_list_partial, verify_full, Partial, 12 | }; 13 | use crate::parser::{tokens, IResult}; 14 | use crate::HasSpan; 15 | 16 | pub fn fn_decl(input: Tokens) -> IResult> { 17 | let simple = map(map(tokens::identifier, Pattern::Simple), Partial::from); 18 | let set = map_partial(set_pattern, Pattern::Set); 19 | let pattern = terminated(alt((simple, set)), tokens::colon); 20 | let expr = alt((expr, util::error_expr_if(tokens::eof))); 21 | let fn_decl = map_partial_spanned(pair_partial(pattern, expr), |span, (pattern, expr)| { 22 | ExprFnDecl::new(pattern, expr, span) 23 | }); 24 | terminated(fn_decl, many0_count(tokens::comment))(input) 25 | } 26 | 27 | fn set_pattern(input: Tokens) -> IResult> { 28 | // Definition: 29 | // a leading token sequence B of a non-terminal A is a production rule of B 30 | // such that if a non-terminal C satisfies the condition that all words in its 31 | // language L(C) has a prefix as a word in L(B), then L(A) = L(C), ie. A and C 32 | // produces the same set of words. 33 | // Note: unfortunately computing a leading token sequence of any non-terminal in any grammar 34 | // is undecidable, but luckily we know one for this. 35 | 36 | // Quickly look for a leading token sequence, because recovery with partial 37 | // parsing result may produces invalid parsing tree on the rest of the tokens 38 | // if there is no sequence of valid leading tokens 39 | 40 | // According to the grammar, the leading token sequence will be 41 | // left-curly-bracket ( right-curly-bracket colon / ellipsis / ident ( question / comma / right-curly-bracket )) 42 | fn ignore(_: T) {} 43 | preceded( 44 | pair(tokens::brace_left, many0_count(tokens::comment)), 45 | alt(( 46 | map(preceded(tokens::brace_right, tokens::colon), ignore), 47 | map(tokens::ellipsis, ignore), 48 | preceded( 49 | tokens::identifier, 50 | alt(( 51 | map(tokens::op_question, ignore), 52 | map(tokens::comma, ignore), 53 | map(tokens::brace_right, ignore), 54 | )), 55 | ), 56 | )), 57 | )(input)?; 58 | 59 | let value = alt((expr, util::error_expr_if(tokens::comma))); 60 | let default = opt(preceded(tokens::op_question, verify_full(value))); 61 | let formal = map(pair(tokens::identifier, default), |(name, def)| { 62 | let name_span = name.span(); 63 | let default_span = def.as_ref().map(|d| d.span()).unwrap_or(name_span); 64 | Partial::from(Formal::new(name, def, Span::merge(name_span, default_span))) 65 | }); 66 | 67 | let sep = pair(tokens::comma, many0_count(tokens::comment)); 68 | let args = separated_list_partial(sep, tokens::brace_right, formal); 69 | let start = pair(tokens::brace_left, many0_count(tokens::comment)); 70 | let term = pair(tokens::brace_right, many0_count(tokens::comment)); 71 | 72 | map_partial_spanned(delimited(start, args, term), |span, formals| { 73 | SetPattern::new(formals, None, None, span) 74 | })(input) 75 | } 76 | 77 | #[cfg(test)] 78 | mod tests { 79 | use super::*; 80 | 81 | use crate::lexer::Lexer; 82 | 83 | #[test] 84 | fn formal_args() { 85 | let source = r#"{foo}:foo"#; 86 | let lexer = Lexer::new(source).unwrap(); 87 | let tokens = lexer.tokens(); 88 | set_pattern(tokens).unwrap(); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /nix-parser/src/parser/expr/stmt.rs: -------------------------------------------------------------------------------- 1 | use nom::branch::alt; 2 | use nom::sequence::preceded; 3 | 4 | use super::{bind, expr, util}; 5 | use crate::ast::{ExprAssert, ExprLetIn, ExprWith}; 6 | use crate::lexer::Tokens; 7 | use crate::parser::partial::{ 8 | expect_terminated, many_till_partial, map_partial_spanned, pair_partial, Partial, 9 | }; 10 | use crate::parser::{tokens, IResult}; 11 | 12 | pub fn with(input: Tokens) -> IResult> { 13 | let delims = alt((tokens::semi, tokens::eof)); 14 | let scope = alt((expr, util::error_expr_if(delims))); 15 | let with = expect_terminated(preceded(tokens::keyword_with, expr), tokens::semi); 16 | let stmt = pair_partial(with, scope); 17 | map_partial_spanned(stmt, |span, (with, body)| ExprWith::new(with, body, span))(input) 18 | } 19 | 20 | pub fn assert(input: Tokens) -> IResult> { 21 | let delims = alt((tokens::semi, tokens::eof)); 22 | let cond = alt((expr, util::error_expr_if(delims))); 23 | let assert = expect_terminated(preceded(tokens::keyword_assert, cond), tokens::semi); 24 | let stmt = pair_partial(assert, expr); 25 | map_partial_spanned(stmt, |span, (cond, body)| ExprAssert::new(cond, body, span))(input) 26 | } 27 | 28 | pub fn let_in(input: Tokens) -> IResult> { 29 | let binds = many_till_partial(bind::bind, tokens::keyword_in); 30 | let let_binds = expect_terminated(preceded(tokens::keyword_let, binds), tokens::keyword_in); 31 | let stmt = pair_partial(let_binds, expr); 32 | map_partial_spanned(stmt, |span, (binds, body)| { 33 | ExprLetIn::new(binds, body, span) 34 | })(input) 35 | } 36 | -------------------------------------------------------------------------------- /nix-parser/src/parser/expr/util.rs: -------------------------------------------------------------------------------- 1 | use codespan::Span; 2 | use nom::combinator::{peek, recognize}; 3 | use nom::error::{ErrorKind, ParseError}; 4 | 5 | use crate::ast::Expr; 6 | use crate::error::{Errors, ExpectedFoundError}; 7 | use crate::lexer::Tokens; 8 | use crate::parser::partial::Partial; 9 | use crate::parser::IResult; 10 | use crate::ToSpan; 11 | 12 | pub fn error_expr_if<'a, F>(token: F) -> impl Fn(Tokens<'a>) -> IResult> 13 | where 14 | F: Fn(Tokens<'a>) -> IResult, 15 | { 16 | move |input| match peek(recognize(&token))(input) { 17 | Err(error) => Err(error), 18 | Ok((remaining, tokens)) => { 19 | let desc = tokens.current().description(); 20 | let span = tokens.current().to_span(); 21 | let mut errors = Errors::new(); 22 | errors.push(ExpectedFoundError::new("expression", desc, span)); 23 | let expr = Partial::with_errors(Some(Expr::Error(span)), errors); 24 | Ok((remaining, expr)) 25 | } 26 | } 27 | } 28 | 29 | /// Vendored version of `nom::multi::fold_many0` except it doesn't require `R: Clone`. 30 | /// 31 | /// This combinator should yield noticeably higher performance for complex types, but is not 32 | /// compatible with other `nom` combinators expecting `Fn` closures. 33 | pub fn fold_many0(f: F, init: R, g: G) -> impl FnOnce(I) -> nom::IResult 34 | where 35 | I: Clone + PartialEq, 36 | F: Fn(I) -> nom::IResult, 37 | G: Fn(R, O) -> R, 38 | E: ParseError, 39 | { 40 | move |i: I| { 41 | let mut res = init; 42 | let mut input = i.clone(); 43 | 44 | loop { 45 | let i_ = input.clone(); 46 | match f(i_) { 47 | Ok((i, o)) => { 48 | // loop trip must always consume (otherwise infinite loops) 49 | if i == input { 50 | return Err(nom::Err::Error(E::from_error_kind(input, ErrorKind::Many0))); 51 | } 52 | 53 | res = g(res, o); 54 | input = i; 55 | } 56 | Err(nom::Err::Error(_)) => { 57 | return Ok((input, res)); 58 | } 59 | Err(e) => { 60 | return Err(e); 61 | } 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /nix-parser/src/parser/partial.rs: -------------------------------------------------------------------------------- 1 | //! Data structures and parser combinators for working with partial values. 2 | 3 | use std::iter::FromIterator; 4 | 5 | use codespan::Span; 6 | use nom::bytes::complete::take; 7 | use nom::combinator::opt; 8 | use nom::sequence::preceded; 9 | use nom::InputLength; 10 | 11 | use super::{tokens, IResult}; 12 | use crate::error::{Error, Errors}; 13 | use crate::lexer::Tokens; 14 | use crate::ToSpan; 15 | 16 | /// A partial value which might contain errors. 17 | /// 18 | /// `Partial` is a data structure which may or may not contain a value, and also may or may not 19 | /// contain errors. It is essentially an [`std::option::Option`] with an associated [`Errors`] 20 | /// stack. 21 | /// 22 | /// [`std::option::Option`]: https://doc.rust-lang.org/std/option/enum.Option.html 23 | /// [`Errors`]: ../error/struct.Errors.html 24 | /// 25 | /// This type is used to accumulate and apply monadic transformations to a possibly incomplete 26 | /// [`Expr`] or [`SourceFile`]. Consumers of `Partial` values can choose to assert that the 27 | /// contained value exists without errors with [`Partial::verify()`], which will consume the 28 | /// `Partial` and transform it into a `Result` which can be handled normally. 29 | /// 30 | /// [`Expr`]: ../ast/enum.Expr.html 31 | /// [`SourceFile`]: ../ast/enum.SourceFile.html 32 | /// [`Partial::verify()`]: #method.verify 33 | /// 34 | /// # Examples 35 | /// 36 | /// ``` 37 | /// use codespan::Span; 38 | /// use nix_parser::error::ExpectedFoundError; 39 | /// use nix_parser::parser::Partial; 40 | /// 41 | /// let mut one = Partial::new(Some(1)); 42 | /// assert!(one.value().is_some()); 43 | /// assert!(!one.has_errors()); 44 | /// 45 | /// let checked = if one.value().filter(|i| **i >= 0).is_some() { 46 | /// one.map(|value| value + 1) 47 | /// } else { 48 | /// let error = ExpectedFoundError::new("positive number", "negative number", Span::initial()); 49 | /// one.extend_errors(std::iter::once(error.into())); 50 | /// one 51 | /// }; 52 | /// 53 | /// // You can inspect the partial value and enumerate the errors. 54 | /// println!("Errors, if any: {:?}", checked.errors()); 55 | /// if let Some(ref partial) = checked.value() { 56 | /// println!("Partial value is: {}", partial); 57 | /// } else { 58 | /// println!("No value found"); 59 | /// } 60 | /// 61 | /// // Or you can assert that the contained value exists and has no errors. 62 | /// assert_eq!(checked.verify(), Ok(2)); 63 | /// ``` 64 | #[derive(Clone, Debug, PartialEq)] 65 | pub struct Partial { 66 | value: Option, 67 | errors: Errors, 68 | } 69 | 70 | impl Partial { 71 | /// Constructs a new `Partial` with the given initial value. 72 | /// 73 | /// # Examples 74 | /// 75 | /// ``` 76 | /// # use nix_parser::parser::Partial; 77 | /// let something: Partial = Partial::new(Some(1)); 78 | /// let nothing: Partial = Partial::from(None); 79 | /// ``` 80 | #[inline] 81 | pub fn new(value: Option) -> Self { 82 | Partial { 83 | value, 84 | errors: Errors::new(), 85 | } 86 | } 87 | 88 | /// Constructs a new `Partial` with the given initial value and a stack of errors. 89 | /// 90 | /// # Examples 91 | /// 92 | /// ``` 93 | /// # use nix_parser::parser::Partial; 94 | /// # use nix_parser::error::{Errors, UnexpectedError}; 95 | /// use codespan::Span; 96 | /// 97 | /// let mut errors = Errors::new(); 98 | /// errors.push(UnexpectedError::new("token", Span::new(3, 4))); 99 | /// 100 | /// let value = Partial::with_errors(Some(1), errors); 101 | /// ``` 102 | #[inline] 103 | pub fn with_errors(value: Option, errors: Errors) -> Self { 104 | Partial { value, errors } 105 | } 106 | 107 | /// Returns whether this partial value contains errors. 108 | /// 109 | /// # Examples 110 | /// 111 | /// ``` 112 | /// # use nix_parser::parser::Partial; 113 | /// let value = Partial::from("example"); 114 | /// assert!(!value.has_errors()); 115 | /// 116 | /// // The line above is equivalent to: 117 | /// assert!(value.errors().is_empty()); 118 | /// ``` 119 | #[inline] 120 | pub fn has_errors(&self) -> bool { 121 | !self.errors.is_empty() 122 | } 123 | 124 | /// Returns a reference to any errors associated with the partial value. 125 | /// 126 | /// # Examples 127 | /// 128 | /// ``` 129 | /// # use nix_parser::parser::Partial; 130 | /// let value = Partial::from("example"); 131 | /// assert_eq!(value.errors().len(), 0); 132 | /// ``` 133 | #[inline] 134 | pub fn errors(&self) -> &Errors { 135 | &self.errors 136 | } 137 | 138 | /// Appends the given error to the error stack contained in this partial value. 139 | /// 140 | /// # Examples 141 | /// 142 | /// ``` 143 | /// # use nix_parser::parser::Partial; 144 | /// use codespan::Span; 145 | /// use nix_parser::error::Error; 146 | /// 147 | /// let mut partial = Partial::from("example"); 148 | /// assert!(!partial.has_errors()); 149 | /// 150 | /// let first = Error::Message(Span::new(1, 3), "oops".into()); 151 | /// let second = Error::Message(Span::new(5, 7), "sorry".into()); 152 | /// partial.extend_errors(vec![first, second]); 153 | /// assert_eq!(partial.errors().len(), 2); 154 | /// ``` 155 | pub fn extend_errors>(&mut self, error: I) { 156 | self.errors.extend(error); 157 | } 158 | 159 | /// Returns the contained partial value, if any. 160 | /// 161 | /// # Examples 162 | /// 163 | /// ``` 164 | /// # use nix_parser::parser::Partial; 165 | /// let nothing: Partial = Partial::from(None); 166 | /// assert_eq!(nothing.value(), None); 167 | /// 168 | /// let something: Partial = Partial::from(1); 169 | /// assert_eq!(something.value(), Some(&1)); 170 | /// ``` 171 | #[inline] 172 | pub fn value(&self) -> Option<&T> { 173 | self.value.as_ref() 174 | } 175 | 176 | /// Maps a `Partial` to `Partial` by applying a function to a contained value. 177 | /// 178 | /// This transformation is applied regardless of whether this `Partial` contains errors. 179 | /// 180 | /// # Examples 181 | /// 182 | /// ```rust 183 | /// # use nix_parser::error::Errors; 184 | /// # use nix_parser::parser::Partial; 185 | /// let value = Partial::from("Hello, world!"); 186 | /// let length = value.map(|s| s.len()); 187 | /// assert_eq!(length.value(), Some(&13)); 188 | /// ``` 189 | #[inline] 190 | pub fn map(self, f: F) -> Partial 191 | where 192 | F: FnOnce(T) -> U, 193 | { 194 | Partial { 195 | value: self.value.map(f), 196 | errors: self.errors, 197 | } 198 | } 199 | 200 | /// Calls `f` if there exists a contained value, otherwise returns the stored errors instead. 201 | /// 202 | /// Any errors produced by `f` are appended to the errors already inside `self`. 203 | /// 204 | /// # Examples 205 | /// 206 | /// Notice in the example below how both the values and errors are accumulated. 207 | /// 208 | /// ``` 209 | /// # use nix_parser::error::{Errors, UnexpectedError}; 210 | /// # use nix_parser::parser::Partial; 211 | /// use codespan::Span; 212 | /// 213 | /// let one = Partial::from(1u32); 214 | /// 215 | /// let mut errors = Errors::new(); 216 | /// errors.push(UnexpectedError::new("token", Span::new(3, 4))); 217 | /// let two = Partial::with_errors(Some(2u32), errors); 218 | /// 219 | /// let three = one.flat_map(|x| two.map(|y| x + y)); 220 | /// 221 | /// assert_eq!(three.value(), Some(&3)); 222 | /// assert_eq!(three.errors().len(), 1); 223 | /// ``` 224 | /// 225 | /// If any partial value in the chain returns a `None`, the final value will be `None`. 226 | /// However, the errors are always accumulated regardless. 227 | /// 228 | /// ``` 229 | /// # use nix_parser::error::{Errors, ExpectedFoundError}; 230 | /// # use nix_parser::parser::Partial; 231 | /// use codespan::Span; 232 | /// 233 | /// let one = Partial::from(1u32); 234 | /// 235 | /// let mut errors = Errors::new(); 236 | /// errors.push(ExpectedFoundError::new("foo", "bar", Span::new(5, 5))); 237 | /// let two: Partial = Partial::with_errors(None, errors); 238 | /// 239 | /// let three = Partial::from(3u32); 240 | /// 241 | /// let four = one.flat_map(|x| two.flat_map(|y| three.map(|z| x + y + z))); 242 | /// 243 | /// assert_eq!(four.value(), None); 244 | /// assert_eq!(four.errors().len(), 1); 245 | /// ``` 246 | #[inline] 247 | pub fn flat_map(mut self, f: F) -> Partial 248 | where 249 | F: FnOnce(T) -> Partial, 250 | { 251 | if let Some(value) = self.value { 252 | let mut partial = f(value); 253 | self.errors.extend(partial.errors); 254 | partial.errors = self.errors; 255 | partial 256 | } else { 257 | Partial::with_errors(None, self.errors) 258 | } 259 | } 260 | 261 | /// Transforms the `Partial` into a `Result`, asserting that the contained value 262 | /// exists and has no errors. 263 | /// 264 | /// # Examples 265 | /// 266 | /// ```rust 267 | /// # use nix_parser::parser::Partial; 268 | /// let partial = Partial::from(123); 269 | /// assert_eq!(partial.verify(), Ok(123)); 270 | /// 271 | /// let partial: Partial = Partial::from(None); 272 | /// assert!(partial.verify().is_err()); 273 | /// ``` 274 | #[inline] 275 | pub fn verify(self) -> Result { 276 | match self.value { 277 | Some(_) if self.has_errors() => Err(self.errors), 278 | Some(value) => Ok(value), 279 | None => Err(self.errors), 280 | } 281 | } 282 | } 283 | 284 | /// Extend the contents of a `Partial>` from an iterator of `Partial`. 285 | impl Extend> for Partial> { 286 | fn extend(&mut self, iter: I) 287 | where 288 | I: IntoIterator>, 289 | { 290 | if let Some(values) = self.value.as_mut() { 291 | for partial in iter { 292 | self.errors.extend(partial.errors); 293 | values.extend(partial.value); 294 | } 295 | } else { 296 | let errors = iter.into_iter().map(|p| p.errors).flatten(); 297 | self.errors.extend(errors); 298 | } 299 | } 300 | } 301 | 302 | impl From for Partial { 303 | fn from(value: T) -> Self { 304 | Partial::new(Some(value)) 305 | } 306 | } 307 | 308 | impl From> for Partial { 309 | fn from(value: Option) -> Self { 310 | Partial::new(value) 311 | } 312 | } 313 | 314 | /// Collect an iterator of `Partial` into a `Partial>`. 315 | impl FromIterator> for Partial> { 316 | fn from_iter(iter: I) -> Self 317 | where 318 | I: IntoIterator>, 319 | { 320 | let iter = iter.into_iter(); 321 | let (lower, upper) = iter.size_hint(); 322 | let init = (Vec::with_capacity(upper.unwrap_or(lower)), Errors::new()); 323 | let (values, errors) = iter.fold(init, |(mut values, mut errors), partial| { 324 | values.extend(partial.value); 325 | errors.extend(partial.errors); 326 | (values, errors) 327 | }); 328 | 329 | Partial::with_errors(Some(values), errors) 330 | } 331 | } 332 | 333 | /// Combinator which runs the given partial parser and then expects on a terminator. 334 | /// 335 | /// If the terminator is missing, an unclosed delimiter error will be appended to the `Partial`, 336 | /// and parsing will be allowed to continue as though the terminator existed. 337 | pub fn expect_terminated<'a, O1, O2, F, G>( 338 | f: F, 339 | term: G, 340 | ) -> impl Fn(Tokens<'a>) -> IResult> 341 | where 342 | F: Fn(Tokens<'a>) -> IResult>, 343 | G: Fn(Tokens<'a>) -> IResult, 344 | { 345 | move |input| { 346 | let (remaining, mut partial) = f(input)?; 347 | match term(remaining) { 348 | Ok((remaining, _)) => Ok((remaining, partial)), 349 | Err(nom::Err::Error(err)) => { 350 | partial.extend_errors(err); 351 | Ok((remaining, partial)) 352 | } 353 | Err(err) => Err(err), 354 | } 355 | } 356 | } 357 | 358 | /// Combinator which behaves like `nom::combinator::map`, except it is a shorthand for: 359 | /// 360 | /// ```rust,ignore 361 | /// map(partial, |partial| partial.map(&f)) 362 | /// ``` 363 | pub fn map_partial<'a, O1, O2, P, F>( 364 | partial: P, 365 | f: F, 366 | ) -> impl Fn(Tokens<'a>) -> IResult> 367 | where 368 | P: Fn(Tokens<'a>) -> IResult>, 369 | F: Fn(O1) -> O2, 370 | { 371 | move |input| { 372 | let (input, partial) = partial(input)?; 373 | Ok((input, partial.map(&f))) 374 | } 375 | } 376 | 377 | /// Combinator which combines the functionality of `map_partial()` and `map_spanned()`. 378 | /// 379 | /// This is like `map_partial()` except it also includes a `Span` based on the consumed input. 380 | pub fn map_partial_spanned<'a, O1, O2, P, F>( 381 | partial: P, 382 | f: F, 383 | ) -> impl Fn(Tokens<'a>) -> IResult> 384 | where 385 | P: Fn(Tokens<'a>) -> IResult>, 386 | F: Fn(Span, O1) -> O2, 387 | { 388 | move |input| { 389 | let (remainder, partial) = partial(input)?; 390 | let span = if remainder.input_len() > 0 { 391 | Span::new(input.to_span().start(), remainder.to_span().start()) 392 | } else { 393 | input.to_span() 394 | }; 395 | Ok((remainder, partial.map(|p| f(span, p)))) 396 | } 397 | } 398 | 399 | /// Combinator which applies the partial parser `f` until the parser `g` produces a result, 400 | /// returning a `Partial>` of the results of `f`. 401 | /// 402 | /// If the terminator is missing, an unclosed delimiter error will be appended to the `Partial`, 403 | /// and parsing will be allowed to continue as through the terminator existed. 404 | pub fn many_till_partial<'a, O1, O2, F, G>( 405 | f: F, 406 | g: G, 407 | ) -> impl Fn(Tokens<'a>) -> IResult>> 408 | where 409 | F: Fn(Tokens<'a>) -> IResult>, 410 | G: Fn(Tokens<'a>) -> IResult, 411 | { 412 | move |input| { 413 | let mut partials = Vec::new(); 414 | let mut errors = Errors::new(); 415 | let mut input = input; 416 | 417 | loop { 418 | match g(input) { 419 | Ok(_) => { 420 | let mut partial: Partial<_> = partials.into_iter().collect(); 421 | partial.extend_errors(errors); 422 | return Ok((input, partial)); 423 | } 424 | Err(nom::Err::Failure(_)) | Err(nom::Err::Error(_)) => match f(input) { 425 | Err(nom::Err::Failure(err)) | Err(nom::Err::Error(err)) => { 426 | if tokens::eof(input).is_ok() { 427 | let partial: Partial<_> = partials.into_iter().collect(); 428 | return Ok((input, partial)); 429 | } else if let Ok((remainder, _)) = take::<_, _, Errors>(1usize)(input) { 430 | errors.extend(err); 431 | input = remainder; 432 | } 433 | } 434 | Err(err) => return Err(err), 435 | Ok((remainder, elem)) => { 436 | partials.push(elem); 437 | input = remainder; 438 | } 439 | }, 440 | Err(err) => return Err(err), 441 | } 442 | } 443 | } 444 | } 445 | 446 | /// Combinator which returns `None` if the given partial parser fails. 447 | /// 448 | /// This combinator behaves like `nom::combinator::opt`, except it also transposes the result from 449 | /// `Option>` to `Partial>`. This transposition allows you to utilize the 450 | /// behavior of `opt` while remaining compatible with other partial combinators like 451 | /// `pair_partial()` and `map_partial()`. 452 | /// 453 | /// # Examples 454 | /// 455 | /// `opt_partial()` is especially handy when combined with `pair_partial()` to create partial 456 | /// parsers with predicates that trigger based on some non-partial condition. For example, compare 457 | /// this regular `nom` parser with its partial equivalent: 458 | /// 459 | /// ```rust,ignore 460 | /// // Regular version 461 | /// pair(f, opt(preceded(sep, g))) 462 | /// 463 | /// // Partial version 464 | /// pair_partial(f, opt_partial(preceded(sep, g))) 465 | /// ``` 466 | pub fn opt_partial<'a, O, F>(f: F) -> impl Fn(Tokens<'a>) -> IResult>> 467 | where 468 | F: Fn(Tokens<'a>) -> IResult>, 469 | { 470 | move |input| { 471 | let (remaining, value) = opt(&f)(input)?; 472 | match value { 473 | Some(partial) => Ok((remaining, partial.map(Some))), 474 | None => Ok((remaining, Partial::new(Some(None)))), 475 | } 476 | } 477 | } 478 | 479 | /// Combinator which gets the result from the first partial parser, then gets the result from the 480 | /// second partial parser, and produces a partial value containing a tuple of the two results. 481 | /// 482 | /// This is effectively shorthand for: 483 | /// 484 | /// ```rust,ignore 485 | /// map(pair(first, second), |(f, g)| f.flat_map(|f| g.map(|g| (f, g)))) 486 | /// ``` 487 | pub fn pair_partial<'a, O1, O2, F, G>( 488 | first: F, 489 | second: G, 490 | ) -> impl Fn(Tokens<'a>) -> IResult> 491 | where 492 | F: Fn(Tokens<'a>) -> IResult>, 493 | G: Fn(Tokens<'a>) -> IResult>, 494 | { 495 | move |input| { 496 | let (input, f) = first(input)?; 497 | let (remaining, g) = second(input)?; 498 | Ok((remaining, f.flat_map(|f| g.map(|g| (f, g))))) 499 | } 500 | } 501 | 502 | /// Combinator which produces a list of partial elements `f` separated by parser `sep`. 503 | /// 504 | /// This parser behaves like `nom::multi::separated_list`, except that it expects some terminator 505 | /// `term` at the end of the list so it knows when to soft-bail. 506 | /// 507 | /// If the terminator is missing, an unclosed delimiter error will be appended to the `Partial`, 508 | /// and parsing will be allowed to continue as through the terminator existed. 509 | /// 510 | /// This parser is essentially shorthand for: 511 | /// 512 | /// ```rust,ignore 513 | /// let (remaining, (first, rest)) = pair(&f, many_till_partial(preceded(sep, &f), term))(input)?; 514 | /// let partial = first.flat_map(|f| rest.map(|r| std::iter::once(f).chain(r).collect())); 515 | /// ``` 516 | pub fn separated_list_partial<'a, O1, O2, O3, F, G, H>( 517 | sep: G, 518 | term: H, 519 | f: F, 520 | ) -> impl Fn(Tokens<'a>) -> IResult>> 521 | where 522 | F: Fn(Tokens<'a>) -> IResult>, 523 | G: Fn(Tokens<'a>) -> IResult, 524 | H: Fn(Tokens<'a>) -> IResult, 525 | { 526 | move |input| { 527 | let mut partials = Vec::new(); 528 | let mut errors = Errors::new(); 529 | let mut input = input; 530 | 531 | match f(input) { 532 | Err(nom::Err::Error(_)) => return Ok((input, partials.into_iter().collect())), 533 | Err(nom::Err::Failure(err)) => { 534 | return Err(nom::Err::Error(err)); 535 | } 536 | Err(err) => return Err(err), 537 | Ok((remaining, partial)) => { 538 | input = remaining; 539 | partials.push(partial); 540 | } 541 | } 542 | 543 | loop { 544 | match term(input) { 545 | Ok(_) => { 546 | let mut partial: Partial<_> = partials.into_iter().collect(); 547 | partial.extend_errors(errors); 548 | return Ok((input, partial)); 549 | } 550 | Err(nom::Err::Failure(_)) | Err(nom::Err::Error(_)) => { 551 | match preceded(&sep, &f)(input) { 552 | Err(nom::Err::Failure(err)) | Err(nom::Err::Error(err)) => { 553 | if tokens::eof(input).is_ok() { 554 | let partial: Partial<_> = partials.into_iter().collect(); 555 | return Ok((input, partial)); 556 | } else if let Ok((remainder, _)) = take::<_, _, Errors>(1usize)(input) { 557 | errors.extend(err); 558 | input = remainder; 559 | } 560 | } 561 | Err(err) => return Err(err), 562 | Ok((remainder, elem)) => { 563 | partials.push(elem); 564 | input = remainder; 565 | } 566 | } 567 | } 568 | Err(err) => return Err(err), 569 | } 570 | } 571 | } 572 | } 573 | 574 | /// Combinator which asserts that a given partial parser produces a value and contains no errors. 575 | pub fn verify_full<'a, O, F>(f: F) -> impl Fn(Tokens<'a>) -> IResult 576 | where 577 | F: Fn(Tokens<'a>) -> IResult>, 578 | { 579 | move |input| { 580 | let (input, partial) = f(input)?; 581 | partial 582 | .verify() 583 | .map(move |value| (input, value)) 584 | .map_err(nom::Err::Error) 585 | } 586 | } 587 | -------------------------------------------------------------------------------- /nix-parser/src/parser/tokens.rs: -------------------------------------------------------------------------------- 1 | use codespan::Span; 2 | use nom::bytes::complete::take; 3 | use nom::error::ErrorKind; 4 | 5 | use super::IResult; 6 | use crate::ast::tokens::{Comment, Ident}; 7 | use crate::error::{Error, Errors, ExpectedFoundError}; 8 | use crate::lexer::{StringFragment, StringKind, Token, Tokens}; 9 | use crate::ToSpan; 10 | 11 | pub fn comment(input: Tokens) -> IResult { 12 | let (remaining, tokens) = take(1usize)(input)?; 13 | match tokens.current() { 14 | Token::Comment(text, _, span) => Ok((remaining, Comment::from((text.as_str(), *span)))), 15 | token => { 16 | let mut errors = Errors::new(); 17 | errors.push(Error::Nom(token.to_span(), ErrorKind::Tag)); 18 | Err(nom::Err::Error(errors)) 19 | } 20 | } 21 | } 22 | 23 | pub fn identifier(input: Tokens) -> IResult { 24 | let (remaining, tokens) = take(1usize)(input)?; 25 | match tokens.current() { 26 | Token::Identifier(text, span) => Ok((remaining, Ident::from((text.to_string(), *span)))), 27 | token => { 28 | let mut errors = Errors::new(); 29 | let expected = "identifier"; 30 | let found = token.description(); 31 | errors.push(ExpectedFoundError::new(expected, found, token.to_span())); 32 | Err(nom::Err::Error(errors)) 33 | } 34 | } 35 | } 36 | 37 | pub fn interpolation(input: Tokens) -> IResult<(&[Token], Span)> { 38 | let (remaining, tokens) = take(1usize)(input)?; 39 | match tokens.current() { 40 | Token::Interpolation(tokens, span) => Ok((remaining, (&tokens[..], *span))), 41 | token => { 42 | let mut errors = Errors::new(); 43 | errors.push(Error::Nom(token.to_span(), ErrorKind::Tag)); 44 | Err(nom::Err::Error(errors)) 45 | } 46 | } 47 | } 48 | 49 | pub fn string(input: Tokens) -> IResult<(&[StringFragment], StringKind, Span)> { 50 | let (remaining, tokens) = take(1usize)(input)?; 51 | match tokens.current() { 52 | Token::String(ref frags, ref kind, ref span) => Ok((remaining, (&frags[..], *kind, *span))), 53 | token => { 54 | let mut errors = Errors::new(); 55 | errors.push(Error::Nom(token.to_span(), ErrorKind::Tag)); 56 | Err(nom::Err::Error(errors)) 57 | } 58 | } 59 | } 60 | 61 | macro_rules! define_simple_tokens { 62 | ($($function:ident => $name:ident $(( $expected:expr ))? ),+) => { 63 | $(define_simple_tokens!(@token $function => $name $(($expected))?);)+ 64 | }; 65 | 66 | (@token $function:ident => $variant:ident) => { 67 | pub fn $function(input: Tokens) -> IResult { 68 | let (remaining, tokens) = take(1usize)(input)?; 69 | match tokens.current() { 70 | Token::$variant(span) => Ok((remaining, *span)), 71 | token => { 72 | let mut errors = Errors::new(); 73 | errors.push(Error::Nom(token.to_span(), ErrorKind::Tag)); 74 | Err(nom::Err::Error(errors)) 75 | } 76 | } 77 | } 78 | }; 79 | 80 | (@token $function:ident => $variant:ident ( $expected:expr )) => { 81 | pub fn $function(input: Tokens) -> IResult { 82 | let (remaining, tokens) = take(1usize)(input)?; 83 | match tokens.current() { 84 | Token::$variant(span) => Ok((remaining, *span)), 85 | token => { 86 | let mut errors = Errors::new(); 87 | let found = token.description(); 88 | errors.push(ExpectedFoundError::new($expected, found, token.to_span())); 89 | Err(nom::Err::Error(errors)) 90 | } 91 | } 92 | } 93 | }; 94 | } 95 | 96 | define_simple_tokens! { 97 | eof => Eof(""), 98 | 99 | keyword_assert => Assert, 100 | keyword_else => Else("keyword `else`"), 101 | keyword_if => If, 102 | keyword_in => In("keyword `in`"), 103 | keyword_inherit => Inherit("keyword `inherit`"), 104 | keyword_let => Let, 105 | keyword_or => Or, 106 | keyword_rec => Rec, 107 | keyword_then => Then("keyword `then`"), 108 | keyword_with => With, 109 | 110 | op_add => Add, 111 | op_sub => Sub, 112 | op_mul => Mul, 113 | op_div => Div, 114 | op_eq => IsEq, 115 | op_neq => NotEq, 116 | op_lt => LessThan, 117 | op_lte => LessThanEq, 118 | op_gt => GreaterThan, 119 | op_gte => GreaterThanEq, 120 | op_and => LogicalAnd, 121 | op_or => LogicalOr, 122 | op_concat => Concat, 123 | op_update => Update, 124 | op_question => Question, 125 | op_imply => Imply, 126 | op_not => Not, 127 | 128 | at => At("at symbol (`@`)"), 129 | colon => Colon("colon"), 130 | comma => Comma("comma"), 131 | ellipsis => Ellipsis("ellipsis (`...`)"), 132 | dot => Dot("dot separator"), 133 | eq => Eq("equals sign"), 134 | brace_left => LBrace, 135 | brace_right => RBrace("right brace"), 136 | bracket_left => LBracket, 137 | bracket_right => RBracket("right bracket"), 138 | paren_left => LParen, 139 | paren_right => RParen("right parentheses"), 140 | semi => Semi("semicolon") 141 | } 142 | -------------------------------------------------------------------------------- /nix-parser2/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nix-parser2" 3 | version = "0.1.0" 4 | authors = ["Eyal Kalderon "] 5 | edition = "2018" 6 | 7 | [features] 8 | default = [] 9 | serialization = ["codespan/serialization", "nix-lexer/serialization"] 10 | 11 | [dependencies] 12 | codespan = "0.9.3" 13 | codespan-lsp = "0.9.3" 14 | codespan-reporting = "0.9.3" 15 | lsp-types = "0.74" 16 | nix-errors = { path = "../nix-errors" } 17 | nix-lexer = { path = "../nix-lexer" } 18 | rowan = "0.9" 19 | smallvec = "1.2" 20 | smol_str = "0.1" 21 | url = "2.1" 22 | -------------------------------------------------------------------------------- /nix-parser2/src/cst.rs: -------------------------------------------------------------------------------- 1 | //! Lossless concrete syntax tree. 2 | 3 | use codespan::Span; 4 | use nix_errors::{Errors, Partial}; 5 | use nix_lexer::{LiteralKind, StringKind, TokenKind}; 6 | use rowan::{GreenNodeBuilder, Language, SmolStr, TextRange}; 7 | 8 | use crate::error::Error; 9 | use crate::ToSpan; 10 | 11 | /// A node in a concrete syntax tree. 12 | /// 13 | /// These nodes correspond to syntactic constructs such as bindings, attribute paths, formal 14 | /// argument patterns, and expressions. 15 | pub type SyntaxNode = rowan::SyntaxNode; 16 | /// A token with span information. 17 | /// 18 | /// These tokens correspond directly to spans in the source text. 19 | pub type SyntaxToken = rowan::SyntaxToken; 20 | /// A type that represents either a `SyntaxNode` or a `SyntaxToken`. 21 | pub type SyntaxElement = rowan::NodeOrToken; 22 | 23 | impl ToSpan for TextRange { 24 | fn to_span(&self) -> Span { 25 | let start = self.start().to_usize() as u32; 26 | let end = self.end().to_usize() as u32; 27 | Span::new(start, end) 28 | } 29 | } 30 | 31 | impl ToSpan for SyntaxNode { 32 | fn to_span(&self) -> Span { 33 | self.text_range().to_span() 34 | } 35 | } 36 | 37 | /// A marker type for identifying Nix concrete syntax trees. 38 | #[doc(hidden)] 39 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 40 | pub enum NixLanguage {} 41 | 42 | impl Language for NixLanguage { 43 | type Kind = SyntaxKind; 44 | 45 | fn kind_from_raw(raw: rowan::SyntaxKind) -> Self::Kind { 46 | assert!(raw.0 <= SyntaxKind::Root as u16); 47 | unsafe { std::mem::transmute::(raw.0) } 48 | } 49 | 50 | fn kind_to_raw(kind: Self::Kind) -> rowan::SyntaxKind { 51 | kind.into() 52 | } 53 | } 54 | 55 | /// A list specifying all possible syntax elements. 56 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 57 | #[repr(u16)] 58 | pub enum SyntaxKind { 59 | // Tokens 60 | /// `# foo` 61 | LineComment = 0, 62 | /// `/* foo */` 63 | BlockComment, 64 | /// Spaces, tabs, and newlines 65 | Whitespace, 66 | /// `foo`, `_`, `let` 67 | Ident, 68 | /// `3.14` 69 | Float, 70 | /// `1234`, `00001` 71 | Integer, 72 | /// `./foo/bar`, `~/foo/bar`, `/foo/bar`, `foo/bar` 73 | Path, 74 | /// ``, `` 75 | PathTemplate, 76 | /// `https://github.com/NixOS/nix` 77 | Uri, 78 | /// `"` 79 | StringTermNormal, 80 | /// `''` 81 | StringTermIndented, 82 | /// Text span enclosed in one or more `StringTerm`s 83 | StringLiteral, 84 | /// `+` 85 | Add, 86 | /// `-` 87 | Sub, 88 | /// `*` 89 | Mul, 90 | /// `/` 91 | Div, 92 | /// `==` 93 | IsEq, 94 | /// `!=` 95 | NotEq, 96 | /// `<` 97 | LessThan, 98 | /// `<=` 99 | LessThanEq, 100 | /// `>` 101 | GreaterThan, 102 | /// `>=` 103 | GreaterThanEq, 104 | /// `&&` 105 | LogicalAnd, 106 | /// `||` 107 | LogicalOr, 108 | /// `++` 109 | Concat, 110 | /// `//` 111 | Update, 112 | /// `?` 113 | Question, 114 | /// `->` 115 | Imply, 116 | /// `!` 117 | Not, 118 | /// `@` 119 | At, 120 | /// `:` 121 | Colon, 122 | /// `,` 123 | Comma, 124 | /// `.` 125 | Dot, 126 | /// `...` 127 | Ellipsis, 128 | /// `=` 129 | Eq, 130 | /// `${` 131 | OpenInterpolate, 132 | /// `{` 133 | OpenBrace, 134 | /// `}` 135 | CloseBrace, 136 | /// `[` 137 | OpenBracket, 138 | /// `]` 139 | CloseBracket, 140 | /// `(` 141 | OpenParen, 142 | /// `)` 143 | CloseParen, 144 | /// `;` 145 | Semi, 146 | /// Any token unknown to the lexer, e.g. `^` 147 | Unknown, 148 | 149 | // Compound nodes 150 | /// `foo one two` 151 | ExprApply, 152 | /// `assert true != false; true` 153 | ExprAssert, 154 | /// `1 + 1`, `true && false`, `[ 1 2 ] ++ [ 3 4 ]` 155 | ExprBinary, 156 | /// An invalid expression. 157 | ExprError, 158 | /// `foo`, `true`, `null` 159 | ExprIdent, 160 | /// `if true then "success" else "failure"` 161 | ExprIf, 162 | /// `x: x + y`, `{ x, y }: x + y`, `{ x, y, ... }@foo: x + y` 163 | ExprLambda, 164 | /// `let { foo = "hello"; body = foo; }` 165 | ExprLegacyLet, 166 | /// `let foo = "bar"; in foo` 167 | ExprLetIn, 168 | /// `[ 1 2 3 4 ]` 169 | ExprList, 170 | /// `12`, `4.0`, `./foo/bar`, `http://www.example.com` 171 | ExprLiteral, 172 | /// `(foo)` 173 | ExprParen, 174 | /// `foo.bar`, `foo.bar or "true"` 175 | ExprProj, 176 | /// `rec { foo = "bar"; bar = 123; }` 177 | ExprRec, 178 | /// `{ foo = "hello"; bar = 123; }` 179 | ExprSet, 180 | /// `"foo"`, `''bar''`, `"baz ${quux}"` 181 | ExprString, 182 | /// `-12`, `!15.0` 183 | ExprUnary, 184 | /// `with foo; foo.attr` 185 | ExprWith, 186 | /// `foo."bar".${baz}` 187 | AttrPath, 188 | /// `${foo}` 189 | BareInterpolation, 190 | /// `foo = bar;` 191 | BindingSimple, 192 | /// `inherit foo bar;` 193 | BindingInherit, 194 | /// `inherit (expr) foo bar;` 195 | BindingInheritFrom, 196 | /// `foo`, `foo ? { bar = "baz"; }` 197 | FormalArg, 198 | /// `or "foo"` 199 | OrDefault, 200 | /// `x:`, `{ x }:` 201 | Pattern, 202 | /// `{ x, y, ... }@foo:` 203 | /// ^^^^^^^^^^^^^ 204 | SetPattern, 205 | /// `{ x }@foo:` 206 | /// ^^^^ 207 | SetPatternBind, 208 | /// `${foo}` 209 | StringInterpolation, 210 | /// Top-most node of the concrete syntax tree. 211 | Root, 212 | } 213 | 214 | impl SyntaxKind { 215 | /// Returns whether the given token is a kind of trivia, i.e. whitespace and comments. 216 | #[inline] 217 | pub fn is_trivia(self) -> bool { 218 | self.is_whitespace() || self.is_comment() 219 | } 220 | 221 | /// Returns whether the given token is a comment. 222 | #[inline] 223 | pub fn is_comment(self) -> bool { 224 | match self { 225 | SyntaxKind::LineComment | SyntaxKind::BlockComment => true, 226 | _ => false, 227 | } 228 | } 229 | 230 | /// Returns whether the given token is whitespace. 231 | #[inline] 232 | pub fn is_whitespace(self) -> bool { 233 | match self { 234 | SyntaxKind::Whitespace => true, 235 | _ => false, 236 | } 237 | } 238 | } 239 | 240 | impl From for rowan::SyntaxKind { 241 | fn from(kind: SyntaxKind) -> Self { 242 | Self(kind as u16) 243 | } 244 | } 245 | 246 | impl From for SyntaxKind { 247 | fn from(token: TokenKind) -> Self { 248 | match token { 249 | TokenKind::LineComment => SyntaxKind::LineComment, 250 | TokenKind::BlockComment { .. } => SyntaxKind::BlockComment, 251 | TokenKind::Whitespace => SyntaxKind::Whitespace, 252 | 253 | TokenKind::Ident => SyntaxKind::Ident, 254 | TokenKind::Literal { kind } => match kind { 255 | LiteralKind::Float => SyntaxKind::Float, 256 | LiteralKind::Integer => SyntaxKind::Integer, 257 | LiteralKind::Path { .. } => SyntaxKind::Path, 258 | LiteralKind::PathTemplate { .. } => SyntaxKind::PathTemplate, 259 | LiteralKind::Uri => SyntaxKind::Uri, 260 | }, 261 | TokenKind::StringTerm { kind } => match kind { 262 | StringKind::Normal => SyntaxKind::StringTermNormal, 263 | StringKind::Indented => SyntaxKind::StringTermIndented, 264 | }, 265 | TokenKind::StringLiteral => SyntaxKind::StringLiteral, 266 | 267 | TokenKind::Add => SyntaxKind::Add, 268 | TokenKind::Sub => SyntaxKind::Sub, 269 | TokenKind::Mul => SyntaxKind::Mul, 270 | TokenKind::Div => SyntaxKind::Div, 271 | TokenKind::IsEq => SyntaxKind::IsEq, 272 | TokenKind::NotEq => SyntaxKind::NotEq, 273 | TokenKind::LessThan => SyntaxKind::LessThan, 274 | TokenKind::LessThanEq => SyntaxKind::LessThan, 275 | TokenKind::GreaterThan => SyntaxKind::GreaterThan, 276 | TokenKind::GreaterThanEq => SyntaxKind::GreaterThanEq, 277 | TokenKind::LogicalAnd => SyntaxKind::LogicalAnd, 278 | TokenKind::LogicalOr => SyntaxKind::LogicalOr, 279 | TokenKind::Concat => SyntaxKind::Concat, 280 | TokenKind::Update => SyntaxKind::Update, 281 | TokenKind::Question => SyntaxKind::Question, 282 | TokenKind::Imply => SyntaxKind::Imply, 283 | TokenKind::Not => SyntaxKind::Not, 284 | 285 | TokenKind::At => SyntaxKind::At, 286 | TokenKind::Colon => SyntaxKind::Colon, 287 | TokenKind::Comma => SyntaxKind::Comma, 288 | TokenKind::Dot => SyntaxKind::Dot, 289 | TokenKind::Ellipsis => SyntaxKind::Ellipsis, 290 | TokenKind::Eq => SyntaxKind::Eq, 291 | TokenKind::Interpolate => SyntaxKind::OpenInterpolate, 292 | TokenKind::OpenBrace => SyntaxKind::OpenBrace, 293 | TokenKind::CloseBrace => SyntaxKind::CloseBrace, 294 | TokenKind::OpenBracket => SyntaxKind::OpenBracket, 295 | TokenKind::CloseBracket => SyntaxKind::CloseBracket, 296 | TokenKind::OpenParen => SyntaxKind::OpenParen, 297 | TokenKind::CloseParen => SyntaxKind::CloseParen, 298 | TokenKind::Semi => SyntaxKind::Semi, 299 | 300 | TokenKind::Unknown => SyntaxKind::Unknown, 301 | } 302 | } 303 | } 304 | 305 | /// Constructs a new concrete syntax tree node-by-node. 306 | #[derive(Debug, Default)] 307 | pub(crate) struct CstBuilder { 308 | errors: Errors, 309 | inner: GreenNodeBuilder<'static>, 310 | } 311 | 312 | impl CstBuilder { 313 | /// Constructs a new, empty `CstBuilder`. 314 | pub fn new() -> Self { 315 | CstBuilder::default() 316 | } 317 | 318 | /// Creates a new `SyntaxNode` and makes it current. 319 | pub fn start_node(&mut self, kind: SyntaxKind) { 320 | let kind = NixLanguage::kind_to_raw(kind); 321 | self.inner.start_node(kind) 322 | } 323 | 324 | /// Attaches a new token to the current branch. 325 | pub fn token>(&mut self, kind: TokenKind, text: T) { 326 | let kind = NixLanguage::kind_to_raw(SyntaxKind::from(kind)); 327 | self.inner.token(kind, text.into()) 328 | } 329 | 330 | /// Finishes the current branch and restores the previous branch as current. 331 | pub fn finish_node(&mut self) { 332 | self.inner.finish_node(); 333 | } 334 | 335 | /// Appends an error to the error stack. 336 | pub fn error(&mut self, error: Error) { 337 | self.errors.push(error); 338 | } 339 | 340 | /// Returns the finished concrete syntax tree and a stack of parse errors, if any. 341 | pub fn finish(self) -> Partial { 342 | let green = self.inner.finish(); 343 | let root_node = SyntaxNode::new_root(green); 344 | Partial::with_errors(root_node, self.errors) 345 | } 346 | } 347 | -------------------------------------------------------------------------------- /nix-parser2/src/error.rs: -------------------------------------------------------------------------------- 1 | //! Error types for lexing and parsing. 2 | 3 | use codespan::{FileId, Files, Span}; 4 | use codespan_lsp::byte_span_to_range; 5 | use codespan_reporting::diagnostic::{Diagnostic, Label}; 6 | use lsp_types::Diagnostic as LspDiagnostic; 7 | use nix_errors::{LspResult, ToDiagnostic}; 8 | use nix_lexer::TokenKind; 9 | 10 | /// A list of all valid errors that can be emitted by the parser. 11 | #[derive(Clone, Debug, PartialEq)] 12 | pub enum Error { 13 | /// A path or path template literal contains a trailing slash. 14 | TrailingSlash(Span, TokenKind), 15 | /// A block comment or string literal was left unterminated. 16 | Unterminated(Span, TokenKind), 17 | } 18 | 19 | impl ToDiagnostic> for Error { 20 | /// Converts this error to a `codespan_reporting::diagnostic::Diagnostic`. 21 | fn to_diagnostic>(&self, _: &Files, file_id: FileId) -> Diagnostic { 22 | match *self { 23 | Error::TrailingSlash(ref span, ref kind) => { 24 | let message = format!("{}s cannot have a trailing slash", kind.description()); 25 | Diagnostic::error() 26 | .with_message(message) 27 | .with_labels(vec![Label::primary(file_id, *span)]) 28 | } 29 | Error::Unterminated(ref span, ref kind) => { 30 | let message = format!("unterminated {}", kind.description()); 31 | Diagnostic::error() 32 | .with_message(message) 33 | .with_labels(vec![Label::primary(file_id, *span)]) 34 | } 35 | } 36 | } 37 | } 38 | 39 | impl ToDiagnostic> for Error { 40 | /// Converts this error to a `Result`. 41 | fn to_diagnostic>( 42 | &self, 43 | files: &Files, 44 | file_id: FileId, 45 | ) -> LspResult { 46 | match *self { 47 | Error::TrailingSlash(ref span, ref kind) => { 48 | let message = format!("{}s cannot have a trailing slash", kind.description()); 49 | byte_span_to_range(files, file_id, *span) 50 | .map(|range| LspDiagnostic::new_simple(range, message)) 51 | } 52 | Error::Unterminated(ref span, ref kind) => { 53 | let message = format!("unterminated {}", kind.description()); 54 | byte_span_to_range(files, file_id, *span) 55 | .map(|range| LspDiagnostic::new_simple(range, message)) 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /nix-parser2/src/lexer.rs: -------------------------------------------------------------------------------- 1 | //! Utility functions which wrap and extend `nix-lexer`. 2 | 3 | use nix_errors::{Errors, Partial}; 4 | use nix_lexer::{LiteralKind, Token, TokenKind}; 5 | 6 | use crate::error::Error; 7 | 8 | /// Converts an input string into a sequence of tokens and an optional stack of errors. 9 | pub fn tokenize(input: &str) -> Partial, Error> { 10 | let mut errors = Errors::new(); 11 | 12 | #[rustfmt::skip] 13 | let tokens = nix_lexer::tokenize(input) 14 | .inspect(|token| match token.kind { 15 | TokenKind::BlockComment { terminated: false } => { 16 | errors.push(Error::Unterminated(token.span, token.kind)) 17 | } 18 | TokenKind::Literal { kind: LiteralKind::Path { trailing_slash: true } } => { 19 | errors.push(Error::TrailingSlash(token.span, token.kind)) 20 | } 21 | TokenKind::Literal { kind: LiteralKind::PathTemplate { trailing_slash: true } } => { 22 | errors.push(Error::TrailingSlash(token.span, token.kind)) 23 | } 24 | _ => (), 25 | }) 26 | .collect(); 27 | 28 | Partial::with_errors(tokens, errors) 29 | } 30 | -------------------------------------------------------------------------------- /nix-parser2/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Parser for the Nix programming language. 2 | 3 | pub use crate::error::Error; 4 | 5 | use codespan::Span; 6 | use nix_errors::Partial; 7 | use nix_lexer::Token; 8 | 9 | use crate::cst::{CstBuilder, SyntaxKind, SyntaxNode}; 10 | 11 | pub mod ast; 12 | pub mod cst; 13 | 14 | mod error; 15 | mod lexer; 16 | 17 | /// A trait for converting a value to a `codespan::Span`. 18 | /// 19 | /// This is helpful for getting spanned types from external crates to interoperate with `codespan`. 20 | pub trait ToSpan { 21 | /// Converts the given value to a `Span`. 22 | fn to_span(&self) -> Span; 23 | } 24 | 25 | /// Converts an input string to a concrete syntax tree and a stack of parse errors, if any. 26 | pub fn parse(input: &str) -> Partial { 27 | lexer::tokenize(input).flat_map(|tokens| Parser::from_tokens(tokens).parse()) 28 | } 29 | 30 | struct Parser { 31 | tokens: Vec, 32 | builder: CstBuilder, 33 | } 34 | 35 | impl Parser { 36 | pub fn from_tokens(tokens: Vec) -> Self { 37 | Parser { 38 | tokens, 39 | builder: CstBuilder::new(), 40 | } 41 | } 42 | 43 | pub fn parse(mut self) -> Partial { 44 | self.builder.start_node(SyntaxKind::Root); 45 | // TODO: Implement parser here. 46 | self.builder.finish_node(); 47 | self.builder.finish() 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | let 2 | moz_overlay = import (builtins.fetchTarball https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz); 3 | nixpkgs = import { overlays = [ moz_overlay ]; }; 4 | in 5 | with nixpkgs; 6 | stdenv.mkDerivation { 7 | name = "moz_overlay_shell"; 8 | buildInputs = [ 9 | (nixpkgs.rustChannelOf { channel = "stable"; }).rust 10 | nixpkgs.curl 11 | nixpkgs.jq 12 | nixpkgs.sqlite 13 | ] ++ lib.optionals stdenv.isDarwin [ 14 | nixpkgs.darwin.apple_sdk.frameworks.Security 15 | nixpkgs.darwin.apple_sdk.frameworks.CoreServices 16 | ]; 17 | } 18 | -------------------------------------------------------------------------------- /src/backend.rs: -------------------------------------------------------------------------------- 1 | //! HACK: All of this. 2 | 3 | use std::collections::HashMap; 4 | 5 | use codespan::{FileId, Files}; 6 | use codespan_lsp::{make_lsp_diagnostic, range_to_byte_span}; 7 | use jsonrpc_core::Result as RpcResult; 8 | use log::info; 9 | use nix_parser::ast::SourceFile; 10 | use tokio::sync::Mutex; 11 | use tower_lsp::lsp_types::*; 12 | use tower_lsp::{Client, LanguageServer}; 13 | 14 | #[derive(Debug)] 15 | struct State { 16 | sources: HashMap, 17 | files: Files, 18 | } 19 | 20 | #[derive(Debug)] 21 | pub struct Nix { 22 | state: Mutex, 23 | } 24 | 25 | impl Nix { 26 | pub fn new() -> Self { 27 | Nix { 28 | state: Mutex::new(State { 29 | sources: HashMap::new(), 30 | files: Files::new(), 31 | }), 32 | } 33 | } 34 | } 35 | 36 | #[tower_lsp::async_trait] 37 | impl LanguageServer for Nix { 38 | fn initialize(&self, _: &Client, _: InitializeParams) -> RpcResult { 39 | Ok(InitializeResult { 40 | server_info: None, 41 | capabilities: ServerCapabilities { 42 | text_document_sync: Some(TextDocumentSyncCapability::Kind( 43 | TextDocumentSyncKind::Incremental, 44 | )), 45 | completion_provider: Some(CompletionOptions { 46 | resolve_provider: Some(true), 47 | trigger_characters: Some(vec![".".to_string()]), 48 | work_done_progress_options: WorkDoneProgressOptions::default(), 49 | }), 50 | signature_help_provider: Some(SignatureHelpOptions { 51 | trigger_characters: None, 52 | retrigger_characters: None, 53 | work_done_progress_options: WorkDoneProgressOptions::default(), 54 | }), 55 | hover_provider: Some(true), 56 | document_formatting_provider: Some(true), 57 | document_highlight_provider: Some(true), 58 | document_symbol_provider: Some(true), 59 | workspace_symbol_provider: Some(true), 60 | definition_provider: Some(true), 61 | ..ServerCapabilities::default() 62 | }, 63 | }) 64 | } 65 | 66 | async fn shutdown(&self) -> RpcResult<()> { 67 | Ok(()) 68 | } 69 | 70 | async fn did_open(&self, client: &Client, params: DidOpenTextDocumentParams) { 71 | let mut state = self.state.lock().await; 72 | let id = get_or_insert_source(&mut state, ¶ms.text_document); 73 | let diags = get_diagnostics(&state, ¶ms.text_document.uri, id); 74 | client.publish_diagnostics(params.text_document.uri, diags, None); 75 | } 76 | 77 | async fn did_change(&self, client: &Client, params: DidChangeTextDocumentParams) { 78 | let mut state = self.state.lock().await; 79 | let id = reload_source(&mut state, ¶ms.text_document, params.content_changes); 80 | let diags = get_diagnostics(&state, ¶ms.text_document.uri, id); 81 | client.publish_diagnostics(params.text_document.uri, diags, None); 82 | } 83 | } 84 | 85 | fn get_or_insert_source(state: &mut State, document: &TextDocumentItem) -> FileId { 86 | if let Some(id) = state.sources.get(&document.uri) { 87 | *id 88 | } else { 89 | let id = state 90 | .files 91 | .add(document.uri.to_string(), document.text.clone()); 92 | state.sources.insert(document.uri.clone(), id); 93 | id 94 | } 95 | } 96 | 97 | fn reload_source( 98 | state: &mut State, 99 | document: &VersionedTextDocumentIdentifier, 100 | changes: Vec, 101 | ) -> FileId { 102 | if let Some(id) = state.sources.get(&document.uri) { 103 | let mut source = state.files.source(*id).to_owned(); 104 | for change in changes { 105 | if let (None, None) = (change.range, change.range_length) { 106 | source = change.text; 107 | } else if let Some(range) = change.range { 108 | let span = range_to_byte_span(&state.files, *id, &range).unwrap_or_default(); 109 | let range = (span.start().to_usize())..(span.end().to_usize()); 110 | source.replace_range(range, &change.text); 111 | } 112 | } 113 | state.files.update(*id, source); 114 | *id 115 | } else { 116 | panic!("attempted to reload source that does not exist"); 117 | } 118 | } 119 | 120 | fn get_diagnostics(state: &State, uri: &Url, id: FileId) -> Vec { 121 | let source = state.files.source(id); 122 | match source.parse::() { 123 | Ok(expr) => { 124 | info!("parsed expression: {}", expr); 125 | Vec::new() 126 | } 127 | Err(err) => { 128 | info!("expression has errors: {}", err); 129 | err.to_diagnostics(id) 130 | .into_iter() 131 | .map(|d| make_lsp_diagnostic(&state.files, None, d, |_| Ok(uri.clone()))) 132 | .collect::, _>>() 133 | .unwrap() 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![forbid(unsafe_code)] 2 | 3 | use log::info; 4 | use structopt::StructOpt; 5 | use tower_lsp::{LspService, Server}; 6 | 7 | use crate::backend::Nix; 8 | 9 | mod backend; 10 | 11 | pub type Error = Box; 12 | 13 | #[derive(Debug, StructOpt)] 14 | pub struct Args { 15 | /// Enable interactive mode 16 | #[structopt(short = "i", long = "interactive")] 17 | interactive: bool, 18 | } 19 | 20 | pub async fn run(_args: Args) { 21 | env_logger::init(); 22 | info!("Nix Language Server {}", env!("CARGO_PKG_VERSION")); 23 | 24 | let stdin = tokio::io::stdin(); 25 | let stdout = tokio::io::stdout(); 26 | 27 | let (service, messages) = LspService::new(Nix::new()); 28 | Server::new(stdin, stdout) 29 | .interleave(messages) 30 | .serve(service) 31 | .await; 32 | } 33 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use nix_language_server::{self, Args}; 2 | use structopt::StructOpt; 3 | 4 | #[tokio::main] 5 | async fn main() { 6 | let args = Args::from_args(); 7 | nix_language_server::run(args).await; 8 | } 9 | --------------------------------------------------------------------------------