Represent regexp literals explicitly and remove %re syntax

CHANGELOG.md

@@ -14,6 +14,7 @@
 
 #### :boom: Breaking Change
 
+- Remove `%re`; use regexp literals instead. https://github.com/rescript-lang/rescript/pull/8610
 - Reject malformed UTF-8 in documentation comments and invalid string or template literal escapes that were previously accepted, including empty or out-of-range braced Unicode escapes (`\u{}`, `\u{110000}`) and legacy decimal or octal escapes in templates (`\1`, `\01`, `\8`). These inputs now produce syntax diagnostics instead of compiling to invalid or inconsistent JavaScript. https://github.com/rescript-lang/rescript/pull/8606
 - Reject tagged template literals in patterns. Patterns cannot invoke their tag; previously their raw payload was compiled as a plain string comparison. https://github.com/rescript-lang/rescript/pull/8606
 - Remove runtime APIs that were deprecated for removal in ReScript 13, including the `Char` module, unsafe `Obj` operations, legacy `Pervasives` helpers, and `Array.unsafe_get`. https://github.com/rescript-lang/rescript/pull/8564
@@ -37,6 +38,7 @@
 #### :bug: Bug fix
 
 - Fix constant folding of pattern matches on unboxed variants whose payload overlaps a literal constructor, so inlined calls agree with runtime matching. Reject multi-argument unboxed constructors instead of crashing. https://github.com/rescript-lang/rescript/pull/8631
+- Preserve JSX closing tags when recovering from missing prop values. https://github.com/rescript-lang/rescript/pull/8610
 - Fix escaped backticks and interpolation openers in backquoted `%raw`, `%ffi`, and `%re` payloads leaking into emitted JavaScript. https://github.com/rescript-lang/rescript/pull/8630
 - Fix the side-effect analysis treating bigint exponentiation and bounds-checked array and string reads as pure, which let dead-code elimination drop an unused one that throws: `let _ = 2n ** -1n` no longer raised. https://github.com/rescript-lang/rescript/pull/8617
 - Preserve record field `@as` annotations when formatting object types containing spreads. https://github.com/rescript-lang/rescript/pull/8619
@@ -79,6 +81,7 @@
 
 #### :house: Internal
 
+- Give regexp literals a dedicated AST node. https://github.com/rescript-lang/rescript/pull/8610
 - Developer playground: Make panes resizable with wrapping text. https://github.com/rescript-lang/rescript/pull/8628
 - Normalize Lambda terms where they are built: a match guard stays structured data until its fallthrough is known, and `apply` and `mk_builtin` go through the folding constructors. https://github.com/rescript-lang/rescript/pull/8615
 - Replace non-escaping local mutable blocks with scalar bindings when all uses are direct field accesses, generalizing reference unboxing to multi-field records and references captured by JavaScript closures. https://github.com/rescript-lang/rescript/pull/8617

analysis/src/completion_jsx.ml

@@ -284,32 +284,6 @@ type jsx_props = {
   children_start: (int * int) option;
 }
 
-(**
-<div muted= />
-
-This is a special case for JSX props, where the above code is parsed
-as <div muted=//, a regexp literal. We leverage that fact to trigger completion
-for the JSX prop value.
-
-This code is safe because we also check that the location of the expression is broken,
-which only happens when the expression is a parse error/not complete.
-*)
-let is_regexp_jsx_heuristic_expr expr =
-  match expr.Parsetree.pexp_desc with
-  | Pexp_extension
-      ( {txt = "re"},
-        PStr
-          [
-            {
-              pstr_desc =
-                Pstr_eval
-                  ({pexp_desc = Pexp_constant (Pconst_raw_source "//")}, _);
-            };
-          ] )
-    when expr.pexp_loc |> Loc.end_ = (Location.none |> Loc.end_) ->
-    true
-  | _ -> false
-
 let find_jsx_props_completable ~jsx_props ~end_pos ~pos_before_cursor
     ~first_char_before_cursor_no_white ~char_at_cursor ~pos_after_comp_name =
   let all_labels =
@@ -392,14 +366,9 @@ let find_jsx_props_completable ~jsx_props ~end_pos ~pos_before_cursor
       else if prop.exp.pexp_loc |> Loc.end_ = (Location.none |> Loc.end_) then (
         if Debug.verbose () then
           print_endline "[jsx_props_completable]--> Loc is broken";
-        if
-          Completion_expressions.is_expr_hole prop.exp
-          || is_regexp_jsx_heuristic_expr prop.exp
-        then (
+        if Completion_expressions.is_expr_hole prop.exp then (
           if Debug.verbose () then
-            print_endline
-              "[jsx_props_completable]--> Expr was expr hole or regexp literal \
-               heuristic";
+            print_endline "[jsx_props_completable]--> Expr was expr hole";
           Some
             (Cexpression
                {

analysis/src/dump_ast.ml

@@ -267,6 +267,8 @@ and print_expr_item expr ~pos ~indentation =
     ^ ")"
   | Pexp_extension (({txt} as loc), _) ->
     "Pexp_extension(%" ^ (loc |> print_loc_denominator_loc ~pos) ^ txt ^ ")"
+  | Pexp_regexp {pattern; flags} ->
+    "Pexp_regexp(/" ^ pattern ^ "/" ^ flags ^ ")"
   | Pexp_template {source_segments; values} ->
     "Pexp_template(source_segments=["
     ^ String.concat ", " (List.map (fun {Asttypes.txt} -> txt) source_segments)

analysis/src/utils.ml

@@ -118,6 +118,7 @@ let identify_pexp pexp =
   | Pexp_open _ -> "Pexp_open"
   | Pexp_await _ -> "Pexp_await"
   | Pexp_jsx_element _ -> "Pexp_jsx_element"
+  | Pexp_regexp _ -> "Pexp_regexp"
   | Pexp_template _ -> "Pexp_template"
   | Pexp_tagged_template _ -> "Pexp_tagged_template"
 

compiler/ext/config.ml

@@ -2,9 +2,9 @@ let cmi_magic_number = "Caml1999I034"
 
 (* Magic numbers for marshaled values of the *current* parsetree, whose layout
    changes across compiler versions. *)
-and ast_impl_magic_number = "ResImpl01307"
+and ast_impl_magic_number = "ResImpl01308"
 
-and ast_intf_magic_number = "ResIntf01307"
+and ast_intf_magic_number = "ResIntf01308"
 
 (* Magic numbers of the frozen Parsetree0 (OCaml 4.06) layout used on the
    external-PPX wire. They must never be written in front of a

compiler/frontend/ast_exp_extension.ml

@@ -56,11 +56,7 @@ let handle_extension e (_self : Ast_mapper.mapper)
                | Some msg -> " - Todo: " ^ msg)) );
       ]
   | "ffi" -> Ast_exp_handle_external.handle_ffi ~loc ~payload
-  | "raw" -> Ast_exp_handle_external.handle_raw ~kind:Raw_exp loc payload
-  | "re" ->
-    Exp.constraint_ ~loc
-      (Ast_exp_handle_external.handle_raw ~kind:Raw_re loc payload)
-      (Ast_comb.to_regexp_type loc)
+  | "raw" -> Ast_exp_handle_external.handle_raw loc payload
   | "debugger" ->
     {e with pexp_desc = Ast_exp_handle_external.handle_debugger loc payload}
   | _ -> e

compiler/frontend/ast_exp_handle_external.ml

@@ -34,20 +34,13 @@ let handle_debugger loc (payload : Ast_payload.t) =
   | _ ->
     Location.raise_errorf ~loc "%%debugger extension doesn't accept arguments"
 
-let handle_raw ~kind loc payload =
+let handle_raw loc payload =
   let is_function = ref None in
-  match Ast_payload.raw_as_string_exp_exn ~kind ~is_function payload with
-  | None -> (
-    match kind with
-    | Raw_re ->
-      Location.raise_errorf ~loc
-        "%%re extension can only be applied to a string"
-    | Raw_exp ->
-      Location.raise_errorf ~loc
-        "%%raw extension can only be applied to a string"
-    | Raw_program ->
-      Location.raise_errorf ~loc
-        "%%%%raw extension can only be applied to a string")
+  match
+    Ast_payload.raw_as_string_exp_exn ~kind:Raw_exp ~is_function payload
+  with
+  | None ->
+    Location.raise_errorf ~loc "%%raw extension can only be applied to a string"
   | Some exp ->
     {
       exp with

compiler/frontend/ast_exp_handle_external.mli

@@ -26,11 +26,7 @@ val handle_debugger : Location.t -> Ast_payload.t -> Parsetree.expression_desc
 
 val handle_ffi : loc:Location.t -> payload:Ast_payload.t -> Parsetree.expression
 
-val handle_raw :
-  kind:Js_raw_info.raw_kind ->
-  Location.t ->
-  Ast_payload.t ->
-  Parsetree.expression
+val handle_raw : Location.t -> Ast_payload.t -> Parsetree.expression
 
 val handle_raw_structure :
   Location.t -> Ast_payload.t -> Parsetree.structure_item

compiler/frontend/bs_builtin_ppx.ml

@@ -112,6 +112,23 @@ let expr_mapper ~async_context ~in_function_def (self : mapper)
   in_function_def := false;
   match e.pexp_desc with
   (* Its output should not be rewritten anymore *)
+  | Pexp_regexp {pattern; flags} ->
+    let loc = e.pexp_loc in
+    let source = "/" ^ pattern ^ "/" ^ flags in
+    Ast_payload.validate_raw_source ~kind:Raw_re ~loc ~offset:0 source;
+    let raw =
+      Ast_external_mk.local_external_apply loc
+        ~pval_prim:(Prim_name "#raw_expr")
+        ~pval_type:
+          (Ast_helper.Typ.arrow
+             [{attrs = []; lbl = Nolabel; typ = Ast_helper.Typ.any ()}]
+             (Ast_helper.Typ.any ()))
+        [Ast_helper.Exp.constant ~loc (Pconst_raw_source source)]
+    in
+    Ast_helper.Exp.constraint_ ~loc
+      ~attrs:(self.attributes self e.pexp_attributes)
+      {e with pexp_desc = raw; pexp_attributes = []}
+      (Ast_comb.to_regexp_type loc)
   | Pexp_extension extension ->
     Ast_exp_extension.handle_extension e self extension
   | Pexp_constant (Pconst_integer (s, Some 'l')) ->

compiler/ml/ast_helper.ml

@@ -205,6 +205,8 @@ module Exp = struct
   let pack ?loc ?attrs a = mk ?loc ?attrs (Pexp_pack a)
   let open_ ?loc ?attrs a b c = mk ?loc ?attrs (Pexp_open (a, b, c))
   let extension ?loc ?attrs a = mk ?loc ?attrs (Pexp_extension a)
+  let regexp ?loc ?attrs pattern flags =
+    mk ?loc ?attrs (Pexp_regexp {pattern; flags})
   let template ?loc ?attrs source_segments values =
     mk ?loc ?attrs (Pexp_template {source_segments; values})
   let tagged_template ?loc ?attrs tag raw_sources values =

compiler/ml/ast_helper.mli

@@ -226,6 +226,8 @@ module Exp : sig
   val object_literal :
     ?loc:loc -> ?attrs:attrs -> (str * expression) list -> expression
 
+  val regexp : ?loc:loc -> ?attrs:attrs -> string -> string -> expression
+
   val template :
     ?loc:loc -> ?attrs:attrs -> str list -> expression list -> expression
   val letmodule :

compiler/ml/ast_iterator.ml

@@ -390,6 +390,7 @@ module E = struct
       iter_loc sub lid;
       sub.expr sub e
     | Pexp_extension x -> sub.extension sub x
+    | Pexp_regexp _ -> ()
     | Pexp_template {values} -> List.iter (sub.expr sub) values
     | Pexp_tagged_template {tag; values} ->
       sub.expr sub tag;

compiler/ml/ast_mapper.ml

@@ -352,6 +352,7 @@ module E = struct
     | Pexp_for_await_of (p, e1, e2) ->
       Exp.mk ~loc ~attrs
         (Pexp_for_await_of (sub.pat sub p, sub.expr sub e1, sub.expr sub e2))
+    | Pexp_regexp {pattern; flags} -> regexp ~loc ~attrs pattern flags
     | Pexp_template {source_segments; values} ->
       Exp.template ~loc ~attrs
         (List.map (map_loc sub) source_segments)

compiler/ml/ast_mapper_from0.ml

@@ -123,7 +123,7 @@ let map_pattern_constant ~loc = function
   | constant -> map_constant ~loc constant
 
 let is_raw_source_extension = function
-  | "raw" | "ffi" | "re" -> true
+  | "raw" | "ffi" -> true
   | _ -> false
 
 let map_raw_source_payload sub = function
@@ -1132,6 +1132,49 @@ module E = struct
     | Pexp_pack me -> pack ~loc ~attrs (sub.module_expr sub me)
     | Pexp_open (ovf, lid, e) ->
       open_ ~loc ~attrs ovf (map_loc sub lid) (sub.expr sub e)
+    | Pexp_extension ({txt = "re"}, payload) -> (
+      let malformed ~loc =
+        Location.raise_errorf ~loc
+          "A PPX returned a malformed regexp payload. Expected a string \
+           containing one regexp literal."
+      in
+      match payload with
+      | PStr
+          [
+            {
+              pstr_desc =
+                Pstr_eval
+                  ( {
+                      pexp_desc = Pexp_constant (Pconst_string (source, _));
+                      pexp_loc = source_loc;
+                      pexp_attributes = source_attrs;
+                    },
+                    eval_attrs );
+            };
+          ] -> (
+        let env = Parser_env.init_env None source in
+        let (_, expression), errors =
+          Parser_flow.do_parse env Parser_flow.Parse.expression false
+        in
+        match expression with
+        | Flow_ast.Expression.RegExpLiteral {pattern; raw}
+          when errors = [] && Parser_env.Peek.token env = Token.T_EOF ->
+          (* Flow filters unknown flags in its [flags] field. Keep the raw
+             spelling so the bridge never silently changes a PPX's regexp. *)
+          let flags_start = String.length pattern + 2 in
+          let flags =
+            String.sub raw flags_start (String.length raw - flags_start)
+          in
+          (* Payload wrappers disappear at this boundary. Keep the expression's
+             location and transfer both levels of payload attributes to it. *)
+          regexp ~loc
+            ~attrs:
+              (attrs
+              @ sub.attributes sub eval_attrs
+              @ sub.attributes sub source_attrs)
+            pattern flags
+        | _ -> malformed ~loc:(sub.location sub source_loc))
+      | _ -> malformed ~loc)
     | Pexp_extension x -> extension ~loc ~attrs (sub.extension sub x)
     | Pexp_unreachable -> assert false
 end
@@ -1369,6 +1412,9 @@ let default_mapper =
     location = (fun _this l -> l);
     extension =
       (fun this (s, payload) ->
+        if s.txt = "re" then
+          Location.raise_errorf ~loc:(this.location this s.loc)
+            "A PPX returned a regexp extension outside an expression.";
         let payload =
           if is_raw_source_extension s.txt then
             match map_raw_source_payload this payload with

compiler/ml/ast_mapper_to0.ml

@@ -652,6 +652,16 @@ module E = struct
         ~attrs:(for_await_of_attr :: attrs)
         (sub.pat sub pat) start_expr end_expr Asttypes.Upto
         (sub.expr sub body_expr)
+    | Pexp_regexp {pattern; flags} ->
+      (* %re is only a frozen PPX wire encoding, not source syntax. *)
+      extension ~loc ~attrs
+        ( Location.mkloc "re" loc,
+          Pt.PStr
+            [
+              Ast_helper0.Str.eval ~loc
+                (Ast_helper0.Exp.constant ~loc
+                   (Pt.Pconst_string ("/" ^ pattern ^ "/" ^ flags, Some "js")));
+            ] )
     | Pexp_template {source_segments; values} ->
       let segments =
         List.map

compiler/ml/ast_payload.ml

@@ -176,6 +176,31 @@ let constructor_tag_of_payload payload =
             | Some (Lident "undefined") -> Some Pct_undefined
             | Some _ | None -> None)))))
 
+let validate_raw_source ~(kind : Js_raw_info.raw_kind) ?is_function ~loc ~offset
+    str =
+  Bs_flow_ast_utils.check_flow_errors ~loc ~offset
+    (match kind with
+    | Raw_re | Raw_exp ->
+      let ((_loc, expression) as program), errors =
+        let open Parser_flow in
+        let env = Parser_env.init_env None str in
+        do_parse env Parse.expression false
+      in
+      (if kind = Raw_re then
+         match expression with
+         | RegExpLiteral _ -> ()
+         | _ ->
+           Location.raise_errorf ~loc
+             "Syntax error: a valid JS regex literal expected");
+      (match is_function with
+      | Some is_function -> (
+        match Classify_function.classify_exp program with
+        | Js_function {arity; _} -> is_function := Some arity
+        | _ -> ())
+      | None -> ());
+      errors
+    | Raw_program -> snd (Parser_flow.parse_program false None str))
+
 let raw_as_string_exp_exn ~(kind : Js_raw_info.raw_kind) ?is_function (x : t) :
     Parsetree.expression option =
   let string_expression =
@@ -214,28 +239,7 @@ let raw_as_string_exp_exn ~(kind : Js_raw_info.raw_kind) ?is_function (x : t) :
   in
   match string_expression with
   | Some (str, offset, ({pexp_loc = loc} as expression)) ->
-    Bs_flow_ast_utils.check_flow_errors ~loc ~offset
-      (match kind with
-      | Raw_re | Raw_exp ->
-        let ((_loc, expression) as program), errors =
-          let open Parser_flow in
-          let env = Parser_env.init_env None str in
-          do_parse env Parse.expression false
-        in
-        (if kind = Raw_re then
-           match expression with
-           | RegExpLiteral _ -> ()
-           | _ ->
-             Location.raise_errorf ~loc
-               "Syntax error: a valid JS regex literal expected");
-        (match is_function with
-        | Some is_function -> (
-          match Classify_function.classify_exp program with
-          | Js_function {arity; _} -> is_function := Some arity
-          | _ -> ())
-        | None -> ());
-        errors
-      | Raw_program -> snd (Parser_flow.parse_program false None str));
+    validate_raw_source ~kind ?is_function ~loc ~offset str;
     Some {expression with pexp_desc = Pexp_constant (Pconst_raw_source str)}
   | None -> None
 

compiler/ml/ast_payload.mli

@@ -53,6 +53,17 @@ val is_single_int : t -> int option
 val constructor_tag_of_payload : t -> Parsetree.constructor_tag option
 (** The literal denoted by a valid variant-constructor [@as] payload. *)
 
+val validate_raw_source :
+  kind:Js_raw_info.raw_kind ->
+  ?is_function:int option ref ->
+  loc:Location.t ->
+  offset:int ->
+  string ->
+  unit
+
+(** Validate JavaScript source using the same rules as raw extensions.
+    [offset] accounts for delimiters preceding the source in its location. *)
+
 val raw_as_string_exp_exn :
   kind:Js_raw_info.raw_kind ->
   ?is_function:int option ref ->

compiler/ml/depend.ml

@@ -225,6 +225,7 @@ let rec add_expr bv exp =
   | Pexp_apply {funct = e; args = el} ->
     add_expr bv e;
     List.iter (fun (_, e) -> add_expr bv e) el
+  | Pexp_regexp _ -> ()
   | Pexp_template {values} -> List.iter (add_expr bv) values
   | Pexp_tagged_template {tag; values} ->
     add_expr bv tag;