[lexical] Bug Fix: Point.isBefore could return incorrect result due to normalization (#7256)

packages/lexical-selection/src/range-selection.ts

@@ -98,10 +98,18 @@ export function $setBlocksType<T extends ElementNode>(
     prevNode.replace(element, true);
     if (newSelection) {
       if (key === newSelection.anchor.key) {
-        newSelection.anchor.key = element.getKey();
+        newSelection.anchor.set(
+          element.getKey(),
+          newSelection.anchor.offset,
+          newSelection.anchor.type,
+        );
       }
       if (key === newSelection.focus.key) {
-        newSelection.focus.key = element.getKey();
+        newSelection.focus.set(
+          element.getKey(),
+          newSelection.focus.offset,
+          newSelection.focus.type,
+        );
       }
     }
   }

packages/lexical/src/LexicalSelection.ts

@@ -17,6 +17,7 @@ import invariant from 'shared/invariant';
 import {
   $caretFromPoint,
   $caretRangeFromSelection,
+  $comparePointCaretNext,
   $createLineBreakNode,
   $createParagraphNode,
   $createTextNode,
@@ -151,23 +152,12 @@ export class Point {
   }
 
   isBefore(b: PointType): boolean {
-    let aNode = this.getNode();
-    let bNode = b.getNode();
-    const aOffset = this.offset;
-    const bOffset = b.offset;
-
-    if ($isElementNode(aNode)) {
-      const aNodeDescendant = aNode.getDescendantByIndex<ElementNode>(aOffset);
-      aNode = aNodeDescendant != null ? aNodeDescendant : aNode;
+    if (this.key === b.key) {
+      return this.offset < b.offset;
     }
-    if ($isElementNode(bNode)) {
-      const bNodeDescendant = bNode.getDescendantByIndex<ElementNode>(bOffset);
-      bNode = bNodeDescendant != null ? bNodeDescendant : bNode;
-    }
-    if (aNode === bNode) {
-      return aOffset < bOffset;
-    }
-    return aNode.isBefore(bNode);
+    const aCaret = $normalizeCaret($caretFromPoint(this, 'next'));
+    const bCaret = $normalizeCaret($caretFromPoint(b, 'next'));
+    return $comparePointCaretNext(aCaret, bCaret) < 0;
   }
 
   getNode(): LexicalNode {

packages/lexical/src/__tests__/unit/LexicalSelection.test.ts

@@ -14,10 +14,13 @@ import {
   ListNode,
 } from '@lexical/list';
 import {
+  $caretRangeFromSelection,
+  $comparePointCaretNext,
   $createLineBreakNode,
   $createParagraphNode,
   $createRangeSelection,
   $createTextNode,
+  $getCaretInDirection,
   $getRoot,
   $getSelection,
   $isParagraphNode,
@@ -1397,3 +1400,42 @@ describe('Regression #7173', () => {
     });
   });
 });
+
+describe('Regression #3181', () => {
+  initializeUnitTest((testEnv) => {
+    test('Point.isBefore edge case with mixed TextNode & ElementNode and matching descendants', () => {
+      testEnv.editor.update(
+        () => {
+          const paragraph = $createParagraphNode();
+          const targetText = $createTextNode('target').setMode('token');
+          $getRoot()
+            .clear()
+            .append(
+              paragraph.append(
+                $createTextNode('a').setMode('token'),
+                $createTextNode('b').setMode('token'),
+                targetText,
+              ),
+            );
+          const selection = paragraph.select(2, 2);
+          selection.focus.set(targetText.getKey(), 1, 'text');
+          expect(selection).toMatchObject({
+            anchor: {key: paragraph.getKey(), offset: 2, type: 'element'},
+            focus: {key: targetText.getKey(), offset: 1, type: 'text'},
+          });
+          const caretRange = $caretRangeFromSelection(selection);
+          expect(
+            $comparePointCaretNext(
+              // These are no-op when isBefore is correct
+              $getCaretInDirection(caretRange.anchor, 'next'),
+              $getCaretInDirection(caretRange.focus, 'next'),
+            ),
+          ).toBe(-1);
+          expect(selection.anchor.isBefore(selection.focus)).toBe(true);
+          expect(selection.focus.isBefore(selection.anchor)).toBe(false);
+        },
+        {discrete: true},
+      );
+    });
+  });
+});

packages/lexical/src/caret/LexicalCaretUtils.ts

@@ -39,6 +39,7 @@ import {
   type TextNode,
 } from '../nodes/LexicalTextNode';
 import {
+  $comparePointCaretNext,
   $getAdjacentChildCaret,
   $getCaretRange,
   $getChildCaret,
@@ -56,7 +57,7 @@ import {
  * @returns a PointCaret for the point
  */
 export function $caretFromPoint<D extends CaretDirection>(
-  point: PointType,
+  point: Pick<PointType, 'type' | 'key' | 'offset'>,
   direction: D,
 ): PointCaret<D> {
   const {type, key, offset} = point;
@@ -154,10 +155,13 @@ export function $caretRangeFromSelection(
   selection: RangeSelection,
 ): CaretRange {
   const {anchor, focus} = selection;
-  const direction = focus.isBefore(anchor) ? 'previous' : 'next';
+  const anchorCaret = $caretFromPoint(anchor, 'next');
+  const focusCaret = $caretFromPoint(focus, 'next');
+  const direction =
+    $comparePointCaretNext(anchorCaret, focusCaret) <= 0 ? 'next' : 'previous';
   return $getCaretRange(
-    $caretFromPoint(anchor, direction),
-    $caretFromPoint(focus, direction),
+    $getCaretInDirection(anchorCaret, direction),
+    $getCaretInDirection(focusCaret, direction),
   );
 }