Simon Génier pushed to branch location-inside-source at Stefan / Typer

Commits: 87882dea by Simon Génier at 2021-09-28T17:45:35-04:00 Compute location inside the source from the given point.

- - - - -

3 changed files:

- src/lexer.ml - src/prelexer.ml - src/source.ml

Changes:

===================================== src/lexer.ml ===================================== @@ -37,8 +37,8 @@ let unescape str =

let lex_number (source : #Source.t) - (start_location : Source.Location.t) (start_point : Source.Point.t) + (doc : string) : sexp =

let rec lex_exponent () = @@ -48,8 +48,8 @@ let lex_number lex_exponent ()

| _ - -> let source = source#slice start_point in - Float (start_location, float_of_string source) + -> let source, location = source#slice start_point doc in + Float (location, float_of_string source) in

let lex_exponent_sign () = @@ -60,8 +60,8 @@ let lex_number

(* TODO: error if there are no signs or digits after the 'e'? *) | _ - -> let source = source#slice start_point in - Float (start_location, float_of_string source) + -> let source, location = source#slice start_point doc in + Float (location, float_of_string source) in

let rec lex_fractional () = @@ -75,8 +75,8 @@ let lex_number lex_exponent_sign ()

| _ - -> let source = source#slice start_point in - Float (start_location, float_of_string source) + -> let source, location = source#slice start_point doc in + Float (location, float_of_string source) in

let rec lex_integer () = @@ -94,8 +94,8 @@ let lex_number lex_exponent_sign ()

| _ - -> let source = source#slice start_point in - Integer (start_location, Z.of_string source) + -> let source, location = source#slice start_point doc in + Integer (location, Z.of_string source) in

lex_integer () @@ -105,31 +105,30 @@ let lex_symbol (token_env : token_env) (source : #Source.t) (doc : string) - (start_location : Source.Location.t) (start_point : Source.Point.t) : sexp =

- let mksym start_location start_point escaped = + let mksym start_point escaped = if Source.Point.equal start_point source#point - then epsilon start_location + then epsilon (Source.Location.of_point source#file start_point doc) else - let raw_name = source#slice start_point in + let raw_name, location = source#slice start_point doc in let name = if escaped then unescape raw_name else raw_name in - hSymbol (start_location, name) + hSymbol (location, name) in

- let rec loop start_location start_point prec lf (escaped : bool) = + let rec loop start_point prec lf (escaped : bool) = match source#peek with | None - -> lf (mksym start_location start_point escaped) + -> lf (mksym start_point escaped)

| Some c -> (match token_env.(Char.code c) with | CKseparate - -> lf (mksym start_location start_point escaped) + -> lf (mksym start_point escaped)

| CKinner prec' - -> let left = mksym start_location start_point escaped in + -> let left = mksym start_point escaped in let op_location = source#location doc in

(* To be considered `inner`, an operator char needs to have @@ -151,9 +150,9 @@ let lex_symbol then fun s -> lf (Node (op, [left; s])) else fun s -> Node (op, [lf left; s]) in - loop (source#location doc) source#point prec' lf' false + loop source#point prec' lf' false else - loop start_location start_point prec lf escaped + loop start_point prec lf escaped

| CKnormal -> (source#advance; @@ -165,11 +164,11 @@ let lex_symbol leading byte, so it has to count as 1 as well ;-) *) | '\' when not is_last -> source#advance; - loop start_location start_point prec lf true + loop start_point prec lf true

- | _ -> loop start_location start_point prec lf escaped)) + | _ -> loop start_point prec lf escaped)) in - loop start_location start_point 0 (fun s -> s) false + loop start_point 0 (fun s -> s) false

let split_presymbol (token_env : token_env) @@ -182,7 +181,6 @@ let split_presymbol | None -> List.rev acc | Some (c) -> let start_point = source#point in - let start_location = source#location doc in let token = match c with (* A negative number literal, or a symbol starting with '-'. *) @@ -190,21 +188,21 @@ let split_presymbol -> (source#advance; match source#peek with | Some ('0' .. '9') - -> lex_number source start_location start_point + -> lex_number source start_point doc | _ - -> lex_symbol token_env source doc start_location start_point) + -> lex_symbol token_env source doc start_point)

| '0' .. '9' -> (source#advance; - lex_number source start_location start_point) + lex_number source start_point doc)

| _ when token_env.(Char.code c) = CKseparate -> source#advance; - let name = source#slice start_point in - hSymbol (start_location, name) + let name, location = source#slice start_point doc in + hSymbol (location, name)

| _ - -> lex_symbol token_env source doc start_location start_point + -> lex_symbol token_env source doc start_point in loop (token :: acc) in

===================================== src/prelexer.ml ===================================== @@ -61,7 +61,7 @@ end (* FIXME: Handle multiline strings. *)

let rec consume_until_newline (source : #Source.t) : unit = - match source#next_char with + match source#next with | None | Some ('\n') -> () | Some _ -> consume_until_newline source

@@ -96,14 +96,14 @@ let prelex (source : #Source.t) : pretoken list = -> source#advance; let start_point = source#point in consume_until_newline source; - let new_doc = String.trim (source#slice start_point) in - loop ctx acc (doc ^ "\n" ^ new_doc) + let new_doc, _ = source#slice start_point "" in + loop ctx acc (doc ^ "\n" ^ String.trim new_doc)

(* A string. *) | Some ('"') -> source#advance; let rec prestring chars = - match source#next_char with + match source#next with | None | Some ('\n') -> let location = source#location doc in prelexer_error location "Unterminated string"; @@ -115,7 +115,7 @@ let prelex (source : #Source.t) : pretoken list = loop ctx (pretoken :: acc) ""

| Some ('\') - -> (match source#next_char with + -> (match source#next with | None | Some ('\n') -> let location = source#location doc in prelexer_error location "Unterminated escape sequence"; @@ -155,25 +155,25 @@ let prelex (source : #Source.t) : pretoken list = let rec pretok () = match source#peek with | None | Some ('%' | '"' | '{' | '}') - -> let location = source#location doc in - let text = source#slice start_point in + -> let text, location = source#slice start_point doc in loop ctx (Pretoken (location, text) :: acc) ""

| Some (c) when c <= ' ' - -> let location = source#location doc in - let text = source#slice start_point in + -> let text, location = source#slice start_point doc in source#advance; loop ctx (Pretoken (location, text) :: acc) ""

| Some ('\') -> source#advance; - (match source#next_char with + (match source#next with | Some _ -> pretok () | None - -> let location = source#location doc in + -> let error_location = source#location doc in source#advance; - let text = source#slice start_point in - prelexer_error location ("Unterminated escape sequence in: " ^ text); + let text, location = source#slice start_point doc in + prelexer_error + error_location + ("Unterminated escape sequence in: " ^ text); loop ctx (Pretoken (location, text) :: acc) "")

| Some _

===================================== src/source.ml ===================================== @@ -33,6 +33,14 @@ module Location = struct

let dummy = {file = ""; line = 0; column = 0; docstr = ""}

+ let of_point + (file : string) + (_, line, column : Point.t) + (docstr : string) + : t = + + {file; line; column; docstr} + let to_string ({file; line; column; _} : t) : string = Printf.sprintf "%s:%d:%d" file line column

@@ -81,7 +89,10 @@ object (self) Note that the source is required only to buffer the last line read and may raise an Invalid_argument if the slice extends before the start of the line. *) - method virtual slice : Point.t -> string + method slice (offset, line, column : Point.t) (docstr : string) : string * Location.t = + (self#slice_impl offset, {file; line; column; docstr}) + + method virtual private slice_impl : int -> string

(* Moves the cursor forward one byte *) method advance : unit = @@ -99,7 +110,7 @@ object (self) method private virtual advance_impl : unit

(* Returns the char at the cursor, then advances it forward. *) - method next_char : char option = + method next : char option = let c = self#peek in self#advance; c @@ -133,12 +144,12 @@ class source_file file_name = object (self) with | End_of_file -> None

- method advance_impl = + method private advance_impl = offset <- offset + 1

method private offset = line_offset + offset

- method slice (start_offset, _, _) = + method private slice_impl start_offset = let relative_start_offset = start_offset - line_offset in String.sub source_line relative_start_offset (offset - relative_start_offset) end @@ -159,11 +170,11 @@ object then Some (source.[offset]) else None

- method advance_impl = + method private advance_impl = offset <- offset + 1

method private offset = offset

- method slice (start_offset, _, _) = + method private slice_impl start_offset = String.sub source start_offset (offset - start_offset) end

View it on GitLab: https://gitlab.com/monnier/typer/-/commit/87882dea3ad0446c118b20b7c61aa3bcb2...