Node.js实现分片上传断点续传示例详解
目录
- 正文
- 文件的分片与合并
- 并发控制
- 使代码可复用
- 服务端接口实现
正文
大文件上传会消耗大量的时间,而且中途有可能上传失败。这时我们需要前端和后端配合来解决这个问题。
解决步骤:
- 文件分片,减少每次请求消耗的时间,如果某次请求失败可以单独上传,而不是从头开始
- 通知服务端合并文件分片
- 控制并发的请求数量,避免浏览器内存溢出
- 当因为网络或者其他原因导致某次的请求失败,我们重新发送请求
文件的分片与合并
在JavaScript中,FIle对象是' Blob '对象的子类,该对象包含一个重要的方法slice,通过该方法我们可以这样分割二进制文件:
<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>Document</title> <script src="https://cdn.bootcdn.net/ajax/libs/axios/0.24.0/axios.min.js"></script> </head> <body> <input type="file" multiple="multiple" id="fileInput" /> <button onclick="SliceUpload()">上传</button> <script> function SliceUpload() { const file = document.getElementById('fileInput').files[0] if (!file) return // 文件分片 let size = 1024 * 50; //50KB 50KB Section size let fileChunks = []; let index = 0; //Section num for (let cur = 0; cur < file.size; cur += size) { fileChunks.push({ hash: index++, chunk: file.slice(cur, cur + size), }); } // 上传分片 const uploadList = fileChunks.map((item, index) => { let formData = new FormData(); formData.append("filename", file.name); formData.append("hash", item.hash); formData.append("chunk", item.chunk); return axios({ method: "post", url: "/upload", data: formData, }); }); await Promise.all(uploadList); // 所有分片上传完成,通知服务器合并分片 await axios({ method: "get", url: "/merge", params: { filename: file.name, }, }); console.log("Upload to complete"); } </script> </body> </html>
并发控制
如果文件很大,这样切分的分片会很多,浏览器短时间内就会发起大量的请求,可能会导致内存耗尽,所以要进行并发控制。
这里我们结合Promise.race()方法 控制并发请求的数量,避免浏览器内存溢出。
// 加入并发控制 async function SliceUpload() { const file = document.getElementById('fileInput').files[0] if (!file) return // 文件分片 let size = 1024 * 50; //50KB 50KB Section size let fileChunks = []; let index = 0; //Section num for (let cur = 0; cur < file.size; cur += size) { fileChunks.push({ hash: index++, chunk: file.slice(cur, cur + size), }); } let pool = []; //Concurrent pool let max = 3; //Maximum concurrency for (let i = 0; i < fileChunks.length; i++) { let item = fileChunks[i]; let formData = new FormData(); formData.append("filename", file.name); formData.append("hash", item.hash); formData.append("chunk", item.chunk); // 上传分片 let task = axios({ method: "post", url: "/upload", data: formData, }); task.then(() => { // 从并发池中移除已经完成的请求 let index = pool.findIndex((t) => t === task); pool.splice(index); }); // 把请求放入并发池中,如果已经达到最大并发量 pool.push(task); if (pool.length === max) { //All requests are requested complete await Promise.race(pool); } } // 所有分片上传完成,通知服务器合并分片 await axios({ method: "get", url: "/merge", params: { filename: file.name, }, }); console.log("Upload to complete"); }
使代码可复用
function SliceUpload() { const file = document.getElementById('fileInput').files[0] if (!file) return // 文件分片 let size = 1024 * 50; // 分片大小设置 let fileChunks = []; let index = 0; // 分片序号 for (let cur = 0; cur < file.size; cur += size) { fileChunks.push({ hash: index++, chunk: file.slice(cur, cur + size), }); } const uploadFileChunks = async function(list){ if(list.length === 0){ // 所有分片上传完成,通知如无 await axios({ method: 'get', url: '/merge', params: { filename: file.name } }); console.log('Upload to complete') return } let pool = [] // 并发池 let max = 3 // 最大并发数 let finish = 0 // 完成数量 let failList = [] // 失败列表 for(let i=0;i<list.length;i++){ let item = list[i] let formData = new FormData() formData.append('filename', file.name) formData.append('hash', item.hash) formData.append('chunk', item.chunk) let task = axios({ method: 'post', url: '/upload', data: formData }) task.then((data)=>{ // 从并发池中移除已经完成的请求 let index = pool.findIndex(t=> t===task) pool.splice(index) }).catch(()=>{ failList.push(item) }).finally(()=>{ finish++ // 如果有失败的重新上传 if(finish===list.length){ uploadFileChunks(failList) } }) pool.push(task) if(pool.length === max){ await Promise.race(pool) } } } uploadFileChunks(fileChunks) }
服务端接口实现
const express = require('express') const multiparty = require('multiparty') const fs = require('fs') const path = require('path') const { Buffer } = require('buffer') // file path const STATIC_FILES = path.join(__dirname, './static/files') // Temporary path to upload files const STATIC_TEMPORARY = path.join(__dirname, './static/temporary') const server = express() // Static file hosting server.use(express.static(path.join(__dirname, './dist'))) // Interface for uploading slices server.post('/upload', (req, res) => { const form = new multiparty.Form(); form.parse(req, function(err, fields, files) { let filename = fields.filename[0] let hash = fields.hash[0] let chunk = files.chunk[0] let dir = `${STATIC_TEMPORARY}/${filename}` // console.log(filename, hash, chunk) try { if (!fs.existsSync(dir)) fs.mkdirSync(dir)
以上就是Node.js实现分片上传示例详解的详细内容,更多关于Node.js分片上传的资料请关注我们其它相关文章!
赞 (0)