diff --git a/clusterview/mode_handlers.py b/clusterview/mode_handlers.py
index ca03b7a..1b52720 100644
--- a/clusterview/mode_handlers.py
+++ b/clusterview/mode_handlers.py
@@ -46,7 +46,7 @@ PointManager.point_set = PointSet(__POINT_SIZE, viewport_height(),
 __left_click_flag = __ClickFlag.NONE
 
 # Variable to track the mouse state during selection movement
-__mouse_start = None
+__last_mouse_pos = None
 
 
 def refresh_point_list(ctx):
@@ -141,13 +141,13 @@ def ogl_keypress_handler(ctx, event):
     """
     global __current_mode
     global __left_click_flag
-    global __mouse_start
+    global __last_mouse_pos
 
     if event.key() == Qt.Key_Escape:
         if ctx.mode is Mode.MOVE:
             if __left_click_flag is not __ClickFlag.NONE:
 
-                __mouse_start = None
+                __last_mouse_pos = None
 
                 __left_click_flag = __ClickFlag.NONE
                 PointManager.point_set.clear_selection()
@@ -173,7 +173,7 @@ def __handle_move_points(ctx, event):
     """
 
     global __left_click_flag
-    global __mouse_start
+    global __last_mouse_pos
 
     set_drawing_event(event)
 
@@ -195,7 +195,7 @@ def __handle_move_points(ctx, event):
             # We are now in the click-and-hold to signal move
             # tracking and translation
             __left_click_flag = __ClickFlag.SELECTION_MOVE
-            __mouse_start = (event.x(), event.y())
+            __last_mouse_pos = (event.x(), event.y())
 
     elif (__left_click_flag is __ClickFlag.SELECTION_BOX
           and event.type() == QEvent.MouseMove):
@@ -203,11 +203,11 @@ def __handle_move_points(ctx, event):
         set_move_bb_bottom_right(event.x(), event.y())
 
     elif (__left_click_flag is __ClickFlag.SELECTION_MOVE
-          and __mouse_start is not None
+          and __last_mouse_pos is not None
           and event.type() == QEvent.MouseMove):
 
-        dx = __mouse_start[0] - event.x()
-        dy = __mouse_start[1] - event.y()
+        dx = abs(__last_mouse_pos[0] - event.x())
+        dy = abs(__last_mouse_pos[1] - event.y())
 
         for p in PointManager.point_set.points:
             if p.selected:
@@ -217,14 +217,15 @@ def __handle_move_points(ctx, event):
                 # fly off screen quickly as we got farther from our
                 # start.
                 try:
-                    if dx > 0:
-                        p.move(1, 0)
-                    if dx < 0:
-                        p.move(-1, 0)
-                    if dy > 0:
-                        p.move(0, 1)
-                    if dy < 0:
-                        p.move(0, -1)
+                    if event.x() < __last_mouse_pos[0]:
+                        p.move(-dx, 0)
+                    if event.y() < __last_mouse_pos[1]:
+                        p.move(0, -dy)
+                    if event.x() > __last_mouse_pos[0]:
+                        p.move(dx, 0)
+                    if event.y() > __last_mouse_pos[1]:
+                        p.move(0, dy)
+
 
                 except ExceededWindowBoundsError:
                     # This point has indicated a move would exceed
@@ -232,6 +233,7 @@ def __handle_move_points(ctx, event):
                     # point.
                     continue
 
+        __last_mouse_pos = (event.x(), event.y())
 
     elif (__left_click_flag is not __ClickFlag.NONE and
           event.type() == QEvent.MouseButtonRelease):