Simon Génier pushed to branch offsets-to-points at Stefan / Typer
Commits: 4fb0ccd3 by Simon Génier at 2021-06-09T12:40:18-04:00 Shorten the names of a few methods on source objects.
These names were redundants. For instance in `current_offset`, the `current` part is already hinted at by the fact that source objects are a kind of cursor. Overall I think the new names a clearer since they are more concise.
- - - - - aff91a51 by Simon Génier at 2021-09-28T17:16:11-04:00 Merge branch 'shorten-source-methods'
- - - - - e8a59f86 by Simon Génier at 2021-09-28T17:25:19-04:00 Replace raw offsets with points which also track line and column.
- - - - -
3 changed files:
- src/lexer.ml - src/prelexer.ml - src/source.ml
Changes:
===================================== src/lexer.ml ===================================== @@ -38,34 +38,34 @@ let unescape str = let lex_number (source : #Source.t) (start_location : Source.Location.t) - (start_offset : int) + (start_point : Source.Point.t) : sexp =
let rec lex_exponent () = - match source#peek_char with + match source#peek with | Some ('0' .. '9') -> source#advance; lex_exponent ()
| _ - -> let source = source#slice_from_offset start_offset in + -> let source = source#slice start_point in Float (start_location, float_of_string source) in
let lex_exponent_sign () = - match source#peek_char with + match source#peek with | Some ('0' .. '9' | '+' | '-') -> source#advance; lex_exponent ()
(* TODO: error if there are no signs or digits after the 'e'? *) | _ - -> let source = source#slice_from_offset start_offset in + -> let source = source#slice start_point in Float (start_location, float_of_string source) in
let rec lex_fractional () = - match source#peek_char with + match source#peek with | Some ('0' .. '9') -> source#advance; lex_fractional () @@ -75,12 +75,12 @@ let lex_number lex_exponent_sign ()
| _ - -> let source = source#slice_from_offset start_offset in + -> let source = source#slice start_point in Float (start_location, float_of_string source) in
let rec lex_integer () = - match source#peek_char with + match source#peek with | Some ('0' .. '9') -> source#advance; lex_integer () @@ -94,7 +94,7 @@ let lex_number lex_exponent_sign ()
| _ - -> let source = source#slice_from_offset start_offset in + -> let source = source#slice start_point in Integer (start_location, Z.of_string source) in
@@ -106,40 +106,40 @@ let lex_symbol (source : #Source.t) (doc : string) (start_location : Source.Location.t) - (start_offset : int) + (start_point : Source.Point.t) : sexp =
- let mksym start_location start_offset escaped = - if Int.equal start_offset source#current_offset + let mksym start_location start_point escaped = + if Source.Point.equal start_point source#point then epsilon start_location else - let raw_name = source#slice_from_offset start_offset in + let raw_name = source#slice start_point in let name = if escaped then unescape raw_name else raw_name in hSymbol (start_location, name) in
- let rec loop start_location start_offset prec lf (escaped : bool) = - match source#peek_char with + let rec loop start_location start_point prec lf (escaped : bool) = + match source#peek with | None - -> lf (mksym start_location start_offset escaped) + -> lf (mksym start_location start_point escaped)
| Some c -> (match token_env.(Char.code c) with | CKseparate - -> lf (mksym start_location start_offset escaped) + -> lf (mksym start_location start_point escaped)
| CKinner prec' - -> let left = mksym start_location start_offset escaped in - let op_location = source#current_location doc in + -> let left = mksym start_location start_point escaped in + let op_location = source#location doc in
(* To be considered `inner`, an operator char needs to have something on its LHS or its RHS, otherwise, treat it as a normal char. `.` can be the normal function composition operator as well. *) - let lhs_p = start_offset != source#current_offset in + let lhs_p = not (Source.Point.equal start_point source#point) in source#advance; let rhs_p = - (match source#peek_char with + (match source#peek with | None -> false | Some c' -> CKseparate != token_env.(Char.code c')) in @@ -151,13 +151,13 @@ let lex_symbol then fun s -> lf (Node (op, [left; s])) else fun s -> Node (op, [lf left; s]) in - loop (source#current_location doc) source#current_offset prec' lf' false + loop (source#location doc) source#point prec' lf' false else - loop start_location start_offset prec lf escaped + loop start_location start_point prec lf escaped
| CKnormal -> (source#advance; - let is_last = Option.is_none source#peek_char in + let is_last = Option.is_none source#peek in match c with (* Skip next char, in case it's a special token. For utf-8, simply advancing is risky but actually works: '' counts as 1 @@ -165,11 +165,11 @@ let lex_symbol leading byte, so it has to count as 1 as well ;-) *) | '\' when not is_last -> source#advance; - loop start_location start_offset prec lf true + loop start_location start_point prec lf true
- | _ -> loop start_location start_offset prec lf escaped)) + | _ -> loop start_location start_point prec lf escaped)) in - loop start_location start_offset 0 (fun s -> s) false + loop start_location start_point 0 (fun s -> s) false
let split_presymbol (token_env : token_env) @@ -178,33 +178,33 @@ let split_presymbol : sexp list =
let rec loop (acc : sexp list) : sexp list = - match source#peek_char with + match source#peek with | None -> List.rev acc | Some (c) - -> let start_offset = source#current_offset in - let start_location = source#current_location doc in + -> let start_point = source#point in + let start_location = source#location doc in let token = match c with (* A negative number literal, or a symbol starting with '-'. *) | '-' -> (source#advance; - match source#peek_char with + match source#peek with | Some ('0' .. '9') - -> lex_number source start_location start_offset + -> lex_number source start_location start_point | _ - -> lex_symbol token_env source doc start_location start_offset) + -> lex_symbol token_env source doc start_location start_point)
| '0' .. '9' -> (source#advance; - lex_number source start_location start_offset) + lex_number source start_location start_point)
| _ when token_env.(Char.code c) = CKseparate -> source#advance; - let name = source#slice_from_offset start_offset in + let name = source#slice start_point in hSymbol (start_location, name)
| _ - -> lex_symbol token_env source doc start_location start_offset + -> lex_symbol token_env source doc start_location start_point in loop (token :: acc) in
===================================== src/prelexer.ml ===================================== @@ -74,7 +74,7 @@ let prelex (source : #Source.t) : pretoken list = (doc : string) : pretoken list =
- match source#peek_char with + match source#peek with | None -> (match ctx with | [] -> List.rev acc @@ -94,9 +94,9 @@ let prelex (source : #Source.t) : pretoken list = (* line's bounds seems ok: String.sub line 1 0 == "" *) | Some ('@') -> source#advance; - let start_offset = source#current_offset in + let start_point = source#point in consume_until_newline source; - let new_doc = String.trim (source#slice_from_offset start_offset) in + let new_doc = String.trim (source#slice start_point) in loop ctx acc (doc ^ "\n" ^ new_doc)
(* A string. *) @@ -105,26 +105,26 @@ let prelex (source : #Source.t) : pretoken list = let rec prestring chars = match source#next_char with | None | Some ('\n') - -> let location = source#current_location doc in + -> let location = source#location doc in prelexer_error location "Unterminated string"; loop ctx (Prestring (location, "") :: acc) ""
| Some ('"') - -> let location = source#current_location doc in + -> let location = source#location doc in let pretoken = Prestring (location, string_implode (List.rev chars)) in loop ctx (pretoken :: acc) ""
| Some ('\') -> (match source#next_char with | None | Some ('\n') - -> let location = source#current_location doc in + -> let location = source#location doc in prelexer_error location "Unterminated escape sequence"; loop ctx (Prestring (location, "") :: acc) "" | Some ('t') -> prestring ('\t' :: chars) | Some ('n') -> prestring ('\n' :: chars) | Some ('r') -> prestring ('\r' :: chars) | Some ('u') - -> let location = source#current_location doc in + -> let location = source#location doc in prelexer_error location "Unimplemented unicode escape"; prestring chars | Some (c) -> prestring (c :: chars)) @@ -134,12 +134,12 @@ let prelex (source : #Source.t) : pretoken list =
| Some ('{') -> source#advance; - let ctx' = (source#current_location doc, acc) :: ctx in + let ctx' = (source#location doc, acc) :: ctx in loop ctx' [] doc
| Some ('}') -> source#advance; - let location = source#current_location doc in + let location = source#location doc in (match ctx with | ((slocation, sacc) :: ctx) -> let preblock = Preblock (slocation, List.rev acc, location) in @@ -150,18 +150,18 @@ let prelex (source : #Source.t) : pretoken list =
(* A pretoken. *) | Some _ - -> let start_offset = source#current_offset in + -> let start_point = source#point in source#advance; let rec pretok () = - match source#peek_char with + match source#peek with | None | Some ('%' | '"' | '{' | '}') - -> let location = source#current_location doc in - let text = source#slice_from_offset start_offset in + -> let location = source#location doc in + let text = source#slice start_point in loop ctx (Pretoken (location, text) :: acc) ""
| Some (c) when c <= ' ' - -> let location = source#current_location doc in - let text = source#slice_from_offset start_offset in + -> let location = source#location doc in + let text = source#slice start_point in source#advance; loop ctx (Pretoken (location, text) :: acc) ""
@@ -170,9 +170,9 @@ let prelex (source : #Source.t) : pretoken list = (match source#next_char with | Some _ -> pretok () | None - -> let location = source#current_location doc in + -> let location = source#location doc in source#advance; - let text = source#slice_from_offset start_offset in + let text = source#slice start_point in prelexer_error location ("Unterminated escape sequence in: " ^ text); loop ctx (Pretoken (location, text) :: acc) "")
===================================== src/source.ml ===================================== @@ -18,6 +18,16 @@ * You should have received a copy of the GNU General Public License along * with this program. If not, see http://www.gnu.org/licenses/. *)
+module Point = struct + (* offset * line * column *) + type t = int * int * int + + let equal (l, _, _ : t) (r, _, _ : t) : bool = + (* The line and column are metadata, only the offset is necessary for + determining equality. *) + Int.equal l r +end + module Location = struct type t = {file : string; line : int; column : int; docstr : string}
@@ -43,37 +53,40 @@ let first_column_of_line = 1 (* A source object is text paired with a cursor. The text can be lazily loaded as it is accessed byte by byte, but it must be retained for future reference by error messages. *) -class virtual t (base_line : int) (base_column : int) (file_name : string) = +class virtual t (base_line : int) (base_column : int) (file : string) = object (self) val mutable line = base_line val mutable column = base_column
(* A path if the text comes from a file, otherwise a meaningful identifier. *) - method file_name : string = file_name + method file : string = file
(* Return the byte at the cursor, or None if the cursor is at the end of the text. *) - method virtual peek_char : char option + method virtual peek : char option
(* The current location of the cursor. *) - method current_location (docstr : string) : Location.t = - {file = file_name; line; column; docstr} + method location (docstr : string) : Location.t = + {file; line; column; docstr} + + method point : Point.t = + (self#offset, line, column)
(* The current offset of the cursor in the file, in bytes. *) - method virtual current_offset : int + method virtual private offset : int
- (* Slices the text, from (and including) a starting offset and to (and + (* Slices the text, from (and including) a starting point and to (and excluding) the curent cursor offset.
Note that the source is required only to buffer the last line read and may raise an Invalid_argument if the slice extends before the start of the line. *) - method virtual slice_from_offset : int -> string + method virtual slice : Point.t -> string
(* Moves the cursor forward one byte *) method advance : unit = let is_utf8_head c = Char.code c < 128 || Char.code c >= 192 in - let c = self#peek_char in + let c = self#peek in self#advance_impl; match c with | Some '\n' @@ -87,7 +100,7 @@ object (self)
(* Returns the char at the cursor, then advances it forward. *) method next_char : char option = - let c = self#peek_char in + let c = self#peek in self#advance; c end @@ -103,29 +116,29 @@ class source_file file_name = object (self) val mutable line_offset = 0 val mutable offset = 0
- method private peek_char_unchecked = + method private peek_unchecked = if offset < String.length source_line then source_line.[offset] else '\n'
- method peek_char = + method peek = if offset <= String.length source_line - then Some self#peek_char_unchecked + then Some self#peek_unchecked else try line_offset <- line_offset + 1 + String.length source_line; source_line <- input_line in_channel; offset <- 0; - Some self#peek_char_unchecked + Some self#peek_unchecked with | End_of_file -> None
method advance_impl = offset <- offset + 1
- method current_offset = line_offset + offset + method private offset = line_offset + offset
- method slice_from_offset start_offset = + method slice (start_offset, _, _) = let relative_start_offset = start_offset - line_offset in String.sub source_line relative_start_offset (offset - relative_start_offset) end @@ -141,7 +154,7 @@ object
val mutable offset = 0
- method peek_char = + method peek = if offset < String.length source then Some (source.[offset]) else None @@ -149,8 +162,8 @@ object method advance_impl = offset <- offset + 1
- method current_offset = offset + method private offset = offset
- method slice_from_offset start_offset = + method slice (start_offset, _, _) = String.sub source start_offset (offset - start_offset) end
View it on GitLab: https://gitlab.com/monnier/typer/-/compare/b6f70b8a97488267c3dc6ebc32f6e6a58...