Fix ConvexHull to return points in counter-clockwise order (#6810)

* Fix ConvexHull to return points in counter-clockwise order

- Add sortCounterClockwise method to ensure CCW ordering
- Start from bottom-most, left-most point for deterministic results
- Fix issue where unordered HashSet broke downstream algorithms
- Add comprehensive tests with CCW order verification

* test(geometry): Achieve 100% test coverage for ConvexHull

src/main/java/com/thealgorithms/geometry/ConvexHull.java

@@ -61,11 +61,24 @@ public final class ConvexHull {
         return new ArrayList<>(convexSet);
     }
 
+    /**
+     * Computes the convex hull using a recursive divide-and-conquer approach.
+     * Returns points in counter-clockwise order starting from the bottom-most, left-most point.
+     *
+     * @param points the input points
+     * @return the convex hull points in counter-clockwise order
+     */
     public static List<Point> convexHullRecursive(List<Point> points) {
+        if (points.size() < 3) {
+            List<Point> result = new ArrayList<>(points);
+            Collections.sort(result);
+            return result;
+        }
+
         Collections.sort(points);
         Set<Point> convexSet = new HashSet<>();
-        Point leftMostPoint = points.get(0);
-        Point rightMostPoint = points.get(points.size() - 1);
+        Point leftMostPoint = points.getFirst();
+        Point rightMostPoint = points.getLast();
 
         convexSet.add(leftMostPoint);
         convexSet.add(rightMostPoint);
@@ -85,9 +98,8 @@ public final class ConvexHull {
         constructHull(upperHull, leftMostPoint, rightMostPoint, convexSet);
         constructHull(lowerHull, rightMostPoint, leftMostPoint, convexSet);
 
-        List<Point> result = new ArrayList<>(convexSet);
-        Collections.sort(result);
-        return result;
+        // Convert to list and sort in counter-clockwise order
+        return sortCounterClockwise(new ArrayList<>(convexSet));
     }
 
     private static void constructHull(Collection<Point> points, Point left, Point right, Set<Point> convexSet) {
@@ -114,4 +126,82 @@ public final class ConvexHull {
             }
         }
     }
+
+    /**
+     * Sorts convex hull points in counter-clockwise order starting from
+     * the bottom-most, left-most point.
+     *
+     * @param hullPoints the unsorted convex hull points
+     * @return the points sorted in counter-clockwise order
+     */
+    private static List<Point> sortCounterClockwise(List<Point> hullPoints) {
+        if (hullPoints.size() <= 2) {
+            Collections.sort(hullPoints);
+            return hullPoints;
+        }
+
+        // Find the bottom-most, left-most point (pivot)
+        Point pivot = hullPoints.getFirst();
+        for (Point p : hullPoints) {
+            if (p.y() < pivot.y() || (p.y() == pivot.y() && p.x() < pivot.x())) {
+                pivot = p;
+            }
+        }
+
+        // Sort other points by polar angle with respect to pivot
+        final Point finalPivot = pivot;
+        List<Point> sorted = new ArrayList<>(hullPoints);
+        sorted.remove(finalPivot);
+
+        sorted.sort((p1, p2) -> {
+            int crossProduct = Point.orientation(finalPivot, p1, p2);
+
+            if (crossProduct == 0) {
+                // Collinear points: sort by distance from pivot (closer first for convex hull)
+                long dist1 = distanceSquared(finalPivot, p1);
+                long dist2 = distanceSquared(finalPivot, p2);
+                return Long.compare(dist1, dist2);
+            }
+
+            // Positive cross product means p2 is counter-clockwise from p1
+            // We want counter-clockwise order, so if p2 is CCW from p1, p1 should come first
+            return -crossProduct;
+        });
+
+        // Build result with pivot first, filtering out intermediate collinear points
+        List<Point> result = new ArrayList<>();
+        result.add(finalPivot);
+
+        if (!sorted.isEmpty()) {
+            // This loop iterates through the points sorted by angle.
+            // For points that are collinear with the pivot, we only want the one that is farthest away.
+            // The sort places closer points first.
+            for (int i = 0; i < sorted.size() - 1; i++) {
+                // Check the orientation of the pivot, the current point, and the next point.
+                int orientation = Point.orientation(finalPivot, sorted.get(i), sorted.get(i + 1));
+
+                // If the orientation is not 0, it means the next point (i+1) is at a new angle.
+                // Therefore, the current point (i) must be the farthest point at its angle. We keep it.
+                if (orientation != 0) {
+                    result.add(sorted.get(i));
+                }
+                // If the orientation is 0, the points are collinear. We discard the current point (i)
+                // because it is closer to the pivot than the next point (i+1).
+            }
+            // Always add the very last point from the sorted list. It is either the only point
+            // at its angle, or it's the farthest among a set of collinear points.
+            result.add(sorted.getLast());
+        }
+
+        return result;
+    }
+
+    /**
+     * Computes the squared distance between two points to avoid floating point operations.
+     */
+    private static long distanceSquared(Point p1, Point p2) {
+        long dx = (long) p1.x() - p2.x();
+        long dy = (long) p1.y() - p2.y();
+        return dx * dx + dy * dy;
+    }
 }