diff --git a/src/List.tsx b/src/List.tsx
index c1b4521b..95b46810 100644
--- a/src/List.tsx
+++ b/src/List.tsx
@@ -271,6 +271,26 @@ export function RawList<T>(props: ListProps<T>, ref: React.Ref<ListRef>) {
   rangeRef.current.start = start;
   rangeRef.current.end = end;
 
+  // When scroll up, first visible item get real height may not same as `itemHeight`,
+  // Which will make scroll jump.
+  // Let's sync scroll top to avoid jump
+  React.useLayoutEffect(() => {
+    const changedRecord = heights.getRecord();
+    if (changedRecord.size === 1) {
+      const recordKey = Array.from(changedRecord)[0];
+      const startIndexKey = getKey(mergedData[start]);
+      if (startIndexKey === recordKey) {
+        const realStartHeight = heights.get(recordKey);
+        const diffHeight = realStartHeight - itemHeight;
+        syncScrollTop((ori) => {
+          return ori + diffHeight;
+        });
+      }
+    }
+
+    heights.resetRecord();
+  }, [scrollHeight]);
+
   // ================================= Size =================================
   const [size, setSize] = React.useState({ width: 0, height });
 
diff --git a/src/hooks/useHeights.tsx b/src/hooks/useHeights.tsx
index 6ab1881c..b4c6955d 100644
--- a/src/hooks/useHeights.tsx
+++ b/src/hooks/useHeights.tsx
@@ -1,5 +1,4 @@
 import findDOMNode from 'rc-util/lib/Dom/findDOMNode';
-import raf from 'rc-util/lib/raf';
 import * as React from 'react';
 import { useEffect, useRef } from 'react';
 import type { GetKey } from '../interface';
@@ -23,10 +22,11 @@ export default function useHeights<T>(
   const [updatedMark, setUpdatedMark] = React.useState(0);
   const instanceRef = useRef(new Map<React.Key, HTMLElement>());
   const heightsRef = useRef(new CacheMap());
-  const collectRafRef = useRef<number>();
+
+  const promiseIdRef = useRef<number>(0);
 
   function cancelRaf() {
-    raf.cancel(collectRafRef.current);
+    promiseIdRef.current += 1;
   }
 
   function collectHeight(sync = false) {
@@ -56,7 +56,13 @@ export default function useHeights<T>(
     if (sync) {
       doCollect();
     } else {
-      collectRafRef.current = raf(doCollect);
+      promiseIdRef.current += 1;
+      const id = promiseIdRef.current;
+      Promise.resolve().then(() => {
+        if (id === promiseIdRef.current) {
+          doCollect();
+        }
+      });
     }
   }
 
diff --git a/src/utils/CacheMap.ts b/src/utils/CacheMap.ts
index a428bbb4..e3d2cef2 100644
--- a/src/utils/CacheMap.ts
+++ b/src/utils/CacheMap.ts
@@ -8,6 +8,8 @@ class CacheMap {
   // `useMemo` no need to update if `id` not change
   id: number = 0;
 
+  diffKeys = new Set<React.Key>();
+
   constructor() {
     this.maps = Object.create(null);
   }
@@ -15,11 +17,24 @@ class CacheMap {
   set(key: React.Key, value: number) {
     this.maps[key as string] = value;
     this.id += 1;
+    this.diffKeys.add(key as string);
   }
 
   get(key: React.Key) {
     return this.maps[key as string];
   }
+
+  /**
+   * CacheMap will record the key changed.
+   * To help to know what's update in the next render.
+   */
+  resetRecord() {
+    this.diffKeys.clear();
+  }
+
+  getRecord() {
+    return this.diffKeys;
+  }
 }
 
 export default CacheMap;