synth: tolerate rbp def/undef merges (one way only)

The easy way of RBP defined vs. undefined merge is implemented (if a
block is defined with RBP undef, we tolerate merging upon it with a
defined RBP by losing this information).

DwarfSynth/Simplest.ml

@@ -654,10 +654,15 @@ let process_blk
 
 exception Inconsistent of BStd.tid
 
-let get_entry_blk graph =
+let get_entry_blk graph first_addr =
+  let filter_out_of_range = function
+    | None -> None
+    | Some x when x < first_addr -> None
+    | Some x -> Some x
+  in
   let entry = BStd.Seq.min_elt (CFG.nodes graph) ~cmp:(fun x y ->
-      let ax = opt_addr_of @@ CFG.Node.label x
-      and ay = opt_addr_of @@ CFG.Node.label y in
+      let ax = filter_out_of_range @@ opt_addr_of @@ CFG.Node.label x
+      and ay = filter_out_of_range @@ opt_addr_of @@ CFG.Node.label y in
       match ax, ay with
       | None, None -> compare x y
       | Some _, None -> -1
@@ -732,14 +737,27 @@ let process_sub sub next_instr_graph : subroutine_cfa_data =
 
   let cfg = BStd.Sub.to_cfg sub in
 
-  let first_addr = int64_addr_of sub in
+  let first_bap_addr = addr_of sub in
+  let first_addr = to_int64_addr first_bap_addr in
   let last_addr = find_last_addr sub in
 
   let initial_cfa_rsp_offset = Int64.of_int 8 in
 
-  let entry_blk = get_entry_blk cfg in
+  let entry_blk = get_entry_blk cfg (first_bap_addr) in
   let rbp_pop_set = find_rbp_pop_set cfg entry_blk in
 
+  let valid_merge previous_regs cur_regs =
+    let valid_rbp_merge old cur =
+      (old = cur) || (match old, cur with
+          | RbpUndef, RbpCfaOffset _ -> true
+          | _ -> false)
+    in
+
+    let prev_cfa, prev_rbp = previous_regs in
+    let cur_cfa, cur_rbp = cur_regs in
+    (prev_cfa = cur_cfa) && valid_rbp_merge prev_rbp cur_rbp
+  in
+
   let rec dfs_process
       allow_rbp
       (sub_changes: (reg_changes_fde * reg_pos) TIdMap.t)
@@ -758,12 +776,18 @@ let process_sub sub next_instr_graph : subroutine_cfa_data =
           allow_rbp entry_offset cur_blk in
       let n_sub_changes =
         TIdMap.add tid (cur_blk_changes, entry_offset) sub_changes in
+
       BStd.Seq.fold (CFG.Node.succs node cfg)
-        ~f:(fun accu child -> dfs_process allow_rbp accu child end_reg)
+        ~f:(fun accu child ->
+            (match entrypoint_address (CFG.Node.label child) with
+             | Some x when x < first_bap_addr -> accu
+             | _ -> dfs_process allow_rbp accu child end_reg)
+            )
+
         ~init:n_sub_changes
     | Some (_, former_entry_offset) ->
       (* Already visited: check that entry values are matching *)
-      if entry_offset <> former_entry_offset then (
+      if not @@ valid_merge former_entry_offset entry_offset then (
         if allow_rbp then
           Format.eprintf "Found inconsistency (0x%Lx <%a>): %a -- %a@."
             (int64_addr_of cur_blk)

csmith/check_generated_eh_frame.py

@@ -59,6 +59,24 @@ def detect_clang_flat_to_pyramid(rows):
       [k'; k[
     """
 
+    def is_flatness_row(row, prev_cfa, prev_loc, first_row=False):
+        for reg in row:
+            if reg not in ["LOC", "CFA", "ra"] and row[reg] != "u":
+                return prev_cfa, prev_loc, True
+        cfa = row["CFA"]
+        if cfa[:4] != "rsp+":
+            return prev_cfa, prev_loc, True
+        cfa_offset = int(cfa[4:])
+        if cfa_offset != prev_cfa + 8:
+            return prev_cfa, prev_loc, True
+        prev_cfa += 8
+        loc = row["LOC"]
+        if not first_row and loc > prev_loc + 2:
+            return prev_cfa, prev_loc, True
+        prev_loc = loc
+
+        return prev_cfa, prev_loc, False
+
     def try_starting_at(start_row):
         if len(rows) < start_row + 1:  # Ensure we have at least the start row
             return rows, False
@@ -69,22 +87,17 @@ def detect_clang_flat_to_pyramid(rows):
         first_cfa = int(rows[start_row]["CFA"][4:])
         prev_cfa = first_cfa
         prev_loc = rows[start_row]["LOC"]
+        first_row = True
+
         for row in rows[start_row + 1 :]:
-            for reg in row:
-                if reg not in ["LOC", "CFA", "ra"] and row[reg] != "u":
-                    break
-            cfa = row["CFA"]
-            if cfa[:4] != "rsp+":
+            prev_cfa, prev_loc, flatness = is_flatness_row(
+                row, prev_cfa, prev_loc, first_row
+            )
+            first_row = False
+            if flatness:
                 break
-            cfa_offset = int(cfa[4:])
-            if cfa_offset != prev_cfa + 8:
-                break
-            prev_cfa += 8
-            loc = row["LOC"]
-            if loc > prev_loc + 2:
-                break
-            prev_loc = loc
             flatness_row_id += 1
+
         flatness_row_id += 1
         if flatness_row_id - start_row <= 1 or flatness_row_id >= len(rows):
             return rows, False  # nothing to change