├── .gitignore
├── package.json
├── README.md
├── main.js
└── src
    ├── worker.js
    ├── virtual-list.html
    └── index.html


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "worker",
 3 |   "version": "1.0.0",
 4 |   "description": "",
 5 |   "main": "main.js",
 6 |   "scripts": {
 7 |     "start": "node main.js"
 8 |   },
 9 |   "keywords": [],
10 |   "author": "",
11 |   "license": "ISC",
12 |   "dependencies": {
13 |     "koa": "^2.13.1",
14 |     "koa-router": "^10.0.0"
15 |   }
16 | }
17 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # 前端海量数据处理
 2 | 
 3 | 针对前端处理海量数据，写了个人的解决方案！
 4 | 
 5 | ## 虚拟列表
 6 | 
 7 | [demo 文件](./src/virtual-list.html)
 8 | 
 9 | <img src='https://p1-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/7a38bac6a3c9436191d08c4ffcfffa5c~tplv-k3u1fbpfcp-watermark.image'>
10 | 
11 | ## 虚拟列表 + webWorker + indexDB 实现
12 | 
13 | <img src='https://p1-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/30e47c192414477db6110427df25975b~tplv-k3u1fbpfcp-watermark.image'>
14 | 
15 | ### 启动步骤：
16 | 
17 | 1. npm i
18 | 
19 | 2. npm start
20 | 
21 | 3. 访问 http://localhost:9527


--------------------------------------------------------------------------------
/main.js:
--------------------------------------------------------------------------------
 1 | const Koa = require('koa')
 2 | const Router = require('koa-router');
 3 | const fs = require('fs/promises')
 4 | 
 5 | const app = new Koa()
 6 | const router = new Router()
 7 | 
 8 | // 实现的虚拟列表
 9 | router.get('/', async (ctx, next) => {
10 |   const content = await fs.readFile('./src/index.html', { encoding: 'utf8' })
11 |   ctx.body = content
12 | })
13 | 
14 | // worker
15 | router.get('/worker.js', async (ctx, next) => {
16 |   const content = await fs.readFile('./src/worker.js')
17 |   ctx.body = content
18 | })
19 | 
20 | const port = 9527
21 | 
22 | app.use(router.routes()).use(router.allowedMethods()).listen(port)
23 |   
24 | console.log(`预览：`, `\x1B[36mhttp://localhost:${port}\x1B[0m`)
25 | 


--------------------------------------------------------------------------------
/src/worker.js:
--------------------------------------------------------------------------------
 1 | const databaseName = 'virtualList'
 2 | let db
 3 | let isFinished = false
 4 | 
 5 | const controlDatabase = db => {
 6 |   db.onversionchange = () => {
 7 |     postMessage({ type: 'close', msg: '只能打开一个页面哦，请关闭其他页面再刷新哦' })
 8 |     db.close()
 9 |   }
10 | }
11 | 
12 | // 创建数据库
13 | const createDatabase = async () => {
14 |   let version = 1
15 |   const databases = await indexedDB.databases()
16 |   const preDatabase = databases.find(({ name }) => name === databaseName)
17 |   if (preDatabase) version = preDatabase.version + 1
18 | 
19 |   // indexedDB.deleteDatabase(databaseName)
20 |   const database = indexedDB.open(databaseName, version)
21 |   return new Promise((f, r) => {
22 |     database.onsuccess = e => controlDatabase(e.target.result)
23 |     database.onupgradeneeded = f
24 |     database.onerror = r
25 |   })
26 | }
27 | 
28 | // 创建表
29 | const createTable = db => {
30 |   return new Promise((f, r) => {
31 |     try {
32 |       db.deleteObjectStore('list')
33 |     }catch (e) {
34 |       console.log(e)
35 |     }
36 |     const table = db.createObjectStore('list', { keyPath: 'id' })
37 |     
38 |     table.transaction.oncomplete = () => {
39 |       const store = db.transaction('list', 'readwrite').objectStore('list')
40 |     
41 |       let i = 0
42 |       while (i < 100000) {
43 |         store.add({ id: i++, num: Math.random() })
44 |       }
45 | 
46 |       f(i)
47 |       isFinished = true
48 |     }
49 |   })
50 | }
51 | 
52 | // 检索
53 | const search = ({ size, start }) => {
54 |   if (!isFinished) return []
55 |   
56 |   return new Promise((f, r) => {
57 |     const store = db.transaction('list', 'readonly').objectStore('list')
58 |     const range = IDBKeyRange.bound(start, start + size)
59 |     const list = []
60 |     
61 |     store.openCursor(range).onsuccess = ({ target: { result } }) => {
62 |       if (!result) {
63 |         return f(list)
64 |       }
65 | 
66 |       list.push(result.value)
67 |       result.continue()
68 |     }
69 |   })
70 | }
71 | 
72 | const start = async () => {
73 |   const { target: { result } } = await createDatabase()
74 |   const data = await createTable(db = result)
75 |   postMessage({
76 |     type: 'ok',
77 |     data
78 |   })
79 | }
80 | 
81 | addEventListener('message', async ({ data }) => {
82 |   const result = await search(data)
83 |   postMessage(result)
84 | })
85 | 
86 | start()


--------------------------------------------------------------------------------
/src/virtual-list.html:
--------------------------------------------------------------------------------
 1 | <html>
 2 |   <head>
 3 |     <meta charset="UTF-8" />
 4 |     <meta name="viewport" content="width=device-width,initial-scale=1.0" />
 5 |     <script src="https://unpkg.com/vue@next"></script>
 6 |     <title>虚拟列表</title>
 7 |     <style>
 8 |       .container {
 9 |         display: flex;
10 |         border: 1px solid;
11 |         overflow-y: auto;
12 |         height: var(--containerHeight);
13 |       }
14 |       .scroll-blank {
15 |         height: var(--height);
16 |       }
17 |       .scroll {
18 |         margin-top: var(--marginTop);
19 |       }
20 |       .scroll-item {
21 |         height: var(--itemHeight);
22 |         /* background-color: pink; */
23 |       }
24 |     </style>
25 |   </head>
26 |   <body>
27 |     <div id="app">
28 |       <div
29 |         ref='container'
30 |         class="container" 
31 |         :style="{
32 |           '--containerHeight': containerHeight + 'px',
33 |           '--height': height + 'px',
34 |           '--itemHeight': itemHeight + 'px',
35 |           '--marginTop': marginTop + 'px',
36 |         }"
37 |       >
38 |         <div class="scroll-blank"></div>
39 |         <div class="scroll">
40 |           <div v-for='(item, index) in activeList' :key='item' class="scroll-item">{{ item }}</div>
41 |         </div>
42 |       </div>
43 |     </div>
44 |     <script>
45 |       const { computed, onMounted, ref } = Vue
46 | 
47 |       const createList = () => {
48 |         const list = []
49 | 
50 |         let i = 0
51 |         while (i < 1000) {
52 |           list.push(i++)
53 |         }
54 | 
55 |         return list
56 |       }
57 |       const App = {
58 |         setup() {
59 |           const container = ref(null)
60 |           const list = createList()
61 |           const containerHeight = 400
62 |           const itemHeight = 20
63 |           const height = list.length * itemHeight
64 |           let start = ref(0)
65 |           let marginTop = ref(0)
66 | 
67 |           const activeList = computed(() => {
68 |             const index = start.value
69 |             return list.slice(index, index + 20)
70 |           })
71 | 
72 |           onMounted(() => {
73 |             container.value.addEventListener('scroll', ({ target }) => {
74 |               const { scrollTop } = target
75 |               const itemNum = scrollTop / itemHeight
76 |               start.value = parseInt(itemNum)
77 |               marginTop.value = scrollTop
78 |             })
79 |           })
80 | 
81 |           return {
82 |             container,
83 |             marginTop,
84 |             containerHeight,
85 |             height,
86 |             itemHeight,
87 |             activeList
88 |           }
89 |         }
90 |       };
91 | 
92 |       const app = Vue.createApp(App);
93 |       app.mount("#app");
94 |     </script>
95 |   </body>
96 | </html>


--------------------------------------------------------------------------------
/src/index.html:
--------------------------------------------------------------------------------
  1 | <html>
  2 |   <head>
  3 |     <meta charset="UTF-8" />
  4 |     <meta name="viewport" content="width=device-width,initial-scale=1.0" />
  5 |     <link rel="stylesheet" href="https://unpkg.com/element-plus/dist/index.css">
  6 |     <script src="https://unpkg.com/vue@next"></script>
  7 |     <script src="https://unpkg.com/element-plus"></script>
  8 |     <title>前端海量数据</title>
  9 |     <style>
 10 |       .container {
 11 |         display: flex;
 12 |         border: 1px solid;
 13 |         overflow-y: auto;
 14 |         height: var(--containerHeight);
 15 |       }
 16 |       .container[loading=true] {
 17 |         pointer-events: none;
 18 |       }
 19 |       .scroll-blank {
 20 |         height: var(--height);
 21 |       }
 22 |       .scroll {
 23 |         margin-top: var(--marginTop);
 24 |       }
 25 |       .scroll-item {
 26 |         height: var(--itemHeight);
 27 |         /* background-color: pink; */
 28 |       }
 29 |     </style>
 30 |   </head>
 31 |   <body>
 32 |     <div id="app">
 33 |       <div
 34 |         v-loading='loading'
 35 |         :loading='loading'
 36 |         ref='container'
 37 |         class="container" 
 38 |         :style="{
 39 |           '--containerHeight': containerHeight + 'px',
 40 |           '--height': height + 'px',
 41 |           '--itemHeight': itemHeight + 'px',
 42 |           '--marginTop': marginTop + 'px',
 43 |         }"
 44 |       >
 45 |         <div class="scroll-blank"></div>
 46 |         <div class="scroll">
 47 |           <div v-for='(item, index) in list' :key='item' class="scroll-item">{{ `${item.id}-------------${item.num}` }}</div>
 48 |         </div>
 49 |       </div>
 50 |     </div>
 51 |     <script>
 52 |       const { watchEffect, onMounted, shallowRef, ref } = Vue
 53 | 
 54 |       const App = {
 55 |         setup() {
 56 |           const container = ref(null)
 57 |           const list = shallowRef([])
 58 |           const height = ref(0)
 59 |           const start = ref(0)
 60 |           const marginTop = ref(0)
 61 |           const loading = ref(true)
 62 |           const isFinished = ref(false)
 63 |           const itemHeight = 20
 64 |           const itemNum = 20
 65 |           const containerHeight = itemHeight * itemNum
 66 | 
 67 |           const worker = new Worker('./worker.js')
 68 | 
 69 |           worker.onmessage = ({ data }) => {
 70 |             if (data.type === 'close') {
 71 |               console.log(data.msg)
 72 |               alert(data.msg)
 73 |               return
 74 |             }
 75 |             if (data.type === 'ok') {
 76 |               isFinished.value = true
 77 |               height.value = data.data * itemHeight
 78 |               return
 79 |             }
 80 |             
 81 |             loading.value = false
 82 |             list.value = data
 83 |           }
 84 | 
 85 |           watchEffect(async () => {
 86 |             if (!isFinished.value) return
 87 |             
 88 |             worker.postMessage({
 89 |               size: itemNum,
 90 |               start: start.value
 91 |             })
 92 |           })
 93 | 
 94 |           onMounted(() => {
 95 |             function fnc () {
 96 |               const { scrollTop } = this
 97 |               const itemNum = scrollTop / itemHeight
 98 |               start.value = parseInt(itemNum)
 99 |               marginTop.value = scrollTop
100 |             }
101 | 
102 |             container.value.addEventListener('scroll', fnc)
103 |           })
104 | 
105 |           return {
106 |             loading,
107 |             list,
108 |             container,
109 |             marginTop,
110 |             containerHeight,
111 |             height,
112 |             itemHeight,
113 |           }
114 |         }
115 |       };
116 | 
117 |       const app = Vue.createApp(App);
118 |       app.use(ElementPlus);
119 |       app.mount("#app");
120 |     </script>
121 |   </body>
122 | </html>


--------------------------------------------------------------------------------