计算机网络🛜 断点续传

前端基石计算机网络 #计算机网络

2021-04-28 | 2024-05-08 | 2.2k | 11 分钟

断点续传-下载

若要实现下载时的断点续传，首先服务器在响应时，要在头中加入下面的字段：
```
Accept-Ranges: bytes
```
这个字段是向客户端表明：这个文件可以支持传输部分数据，只需要告诉我你需要的是哪一部分的数据即可，单位是字节；此时，某些支持断点续传的客户端，比如迅雷，它就可以在请求时，告诉服务器需要的数据范围，具体做法是在请求头中加入下面的字段：
```
range: bytes=0-5000
```
客户端告诉服务器：请给我传递 0-5000 字节范围内的数据即可，无须传输全部数据，完整流程如下：

断点续传-上传

整体来说，实现断点上传的主要思路就是把要上传的文件切分为多个小的数据块然后进行上传
虽然分片上传的整体思路一致，但它没有一个统一的、具体的标准，因此需要根据具体的业务场景制定自己的标准；
由于标准的不同，这也就意味着分片上传需要自行编写代码实现，下面用一种极其简易的流程实现分片上传：

服务端代码

JavaScript

// index.js
const express = require('express');
const path = require('path');
const app = express();
const cors = require('cors');
const port = require('./config').port;

app.use(cors());
app.use('/upload', express.static(path.join(__dirname, './file')));
app.use(express.urlencoded({ extended: true }));
app.use(express.json());

app.get('/download/:filename', (req, res) => {
  const filename = path.join(__dirname, './res', req.params.filename);
  res.download(filename, req.params.filename);
});

app.use('/api/upload', require('./uploader'));

app.listen(port, () => {
  console.log(`server listen on ${port}`);
});

// config.js
module.exports = {
  port: 8000,
};

// uploader.js
const express = require('express');
const router = express.Router();
const file = require('./file');
const config = {
  fieldName: 'file',
  port: require('./config').port,
};
const multer = require('multer');
const storage = multer.memoryStorage();

const upload = multer({
  storage,
}).single(config.fieldName);

router.post('/', upload, async (req, res) => {
  if (!req.body.chunkId) {
    res.send({
      code: 403,
      msg: '请携带分片编号',
      data: null,
    });
    return;
  }
  if (!req.body.fileId) {
    res.send({
      code: 403,
      msg: '请携带文件编号',
      data: null,
    });
    return;
  }
  try {
    const needs = await file.handleChunk(
      req.body.chunkId,
      req.body.fileId,
      req.file.buffer
    );
    res.send({
      code: 0,
      msg: '',
      data: needs,
    });
  } catch (err) {
    res.send({
      code: 403,
      msg: err.message,
      data: null,
    });
  }
});

router.post('/handshake', async (req, res) => {
  if (!req.body.fileId) {
    res.send({
      code: 403,
      msg: '请携带文件编号',
      data: null,
    });
    return;
  }
  if (!req.body.ext) {
    res.send({
      code: 403,
      msg: '请携带文件后缀，例如 .mp4',
      data: null,
    });
    return;
  }
  if (!req.body.chunkIds) {
    res.send({
      code: 403,
      msg: '请按顺序设置文件的分片编号数组',
      data: null,
    });
    return;
  }
  const result = await file.getFileInfo(req.body.fileId, req.body.ext);
  if (result === true) {
    // 不用上传了
    res.send({
      code: 0,
      msg: '',
      data: `${req.protocol}://${req.hostname}:${config.port}/upload/${req.body.fileId}${req.body.ext}`,
    });
    return;
  }
  if (result) {
    // 已经有文件了
    res.send({
      code: 0,
      msg: '',
      data: result.needs,
    });
    return;
  }

  const info = await file.createFileInfo(
    req.body.fileId,
    req.body.ext,
    req.body.chunkIds
  );
  res.send({
    code: 0,
    msg: '',
    data: info.needs,
  });
});

module.exports = router;

// file.js
const fs = require('fs');
const path = require('path');
const chunkDir = path.join(__dirname, './chunktemp');
const fileInfoDir = path.join(__dirname, './filetemp');
const fileDir = path.join(__dirname, './file');

async function exists(path) {
  try {
    await fs.promises.stat(path);
    return true;
  } catch {
    return false;
  }
}

function existsSync(path) {
  try {
    fs.statSync(path);
    return true;
  } catch {
    return false;
  }
}

function createDir() {
  function _createDir(path) {
    if (!existsSync(path)) {
      fs.mkdirSync(path);
    }
  }
  _createDir(chunkDir);
  _createDir(fileInfoDir);
  _createDir(fileDir);
}

createDir();

async function createFileChunk(id, buffer) {
  const absPath = path.join(chunkDir, id);
  if (!(await exists(absPath))) {
    await fs.promises.writeFile(absPath, buffer); // 写入文件
  }
  return {
    id,
    filename: id,
    path: absPath,
  };
}

async function writeFileInfo(id, ext, chunkIds, needs = chunkIds) {
  const absPath = path.join(fileInfoDir, id);
  let info = {
    id,
    ext,
    chunkIds,
    needs,
  };
  await fs.promises.writeFile(absPath, JSON.stringify(info), 'utf-8');
  return info;
}

async function getFileInfo(id) {
  const absPath = path.join(fileInfoDir, id);
  if (!(await exists(absPath))) {
    return null;
  }
  const json = await fs.promises.readFile(absPath, 'utf-8');
  return JSON.parse(json);
}

/**
 * 添加chunk
 */
async function addChunkToFileInfo(chunkId, fileId) {
  const fileInfo = await getFileInfo(fileId);
  if (!fileInfo) {
    return null;
  }
  fileInfo.needs = fileInfo.needs.filter((it) => it !== chunkId);
  return await writeFileInfo(
    fileId,
    fileInfo.ext,
    fileInfo.chunkIds,
    fileInfo.needs
  );
}

async function combine(fileInfo) {
  //1. 将该文件的所有分片依次合并
  const target = path.join(fileDir, fileInfo.id) + fileInfo.ext;

  async function _move(chunkId) {
    const chunkPath = path.join(chunkDir, chunkId);
    const buffer = await fs.promises.readFile(chunkPath);
    await fs.promises.appendFile(target, buffer);
    fs.promises.rm(chunkPath);
  }
  for (const chunkId of fileInfo.chunkIds) {
    await _move(chunkId);
  }

  //2. 删除文件信息
  fs.promises.rm(path.join(fileInfoDir, fileInfo.id));
}

/**
 * return:
 * null: 没有此文件，也没有文件信息
 * true: 有此文件，无须重新上传
 * object：没有此文件，但有该文件的信息
 */
exports.getFileInfo = async function (id, ext) {
  const absPath = path.join(fileDir, id) + ext;
  if (await exists(absPath)) {
    return true;
  }
  return await getFileInfo(id);
};
exports.createFileInfo = async function (id, ext, chunkIds) {
  return await writeFileInfo(id, ext, chunkIds);
};
exports.handleChunk = async function (chunkId, fileId, chunkBuffer) {
  let fileInfo = await getFileInfo(fileId);
  if (!fileInfo) {
    throw new Error('请先提交文件分片信息');
  }
  if (!fileInfo.chunkIds.includes(chunkId)) {
    throw new Error('该文件没有此分片信息');
  }
  if (!fileInfo.needs.includes(chunkId)) {
    // 此分片已经上传
    return fileInfo.needs;
  }
  // 处理分片
  await createFileChunk(chunkId, chunkBuffer);
  // 添加分片信息到文件信息
  fileInfo = await addChunkToFileInfo(chunkId, fileId);
  // 还有需要的分片吗？
  if (fileInfo.needs.length > 0) {
    return fileInfo.needs;
  } else {
    // 全部传完了
    await combine(fileInfo);
    return [];
  }
};

客户端代码

HTML

JavaScript

<div class="container">
  <div class="item">
    <button class="btn choose">选择文件</button>
    <input type="file" id="file" style="display: none" />
    <div class="progress" style="display: none">
      <div class="wrapper">
        <div class="inner">
          <span>0%</span>
        </div>
      </div>
    </div>
    <button
      class="btn control"
      data-status="unchoose"
      style="display: none"
    >
      开始上传
    </button>
  </div>
  <div class="item" id="link" style="display: none">
    <span>文件访问地址：</span>
    <p>
      <a href=""></a>
    </p>
  </div>
</div>
<div class="modal" style="display: none">
  <div class="center">处理中, 请稍后...</div>
</div>
<script src="https://cdn.bootcdn.net/ajax/libs/spark-md5/3.0.0/spark-md5.js"></script>
<script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.6.0/jquery.js"></script>
<script src="./upload.js"></script>

// upload.js
var domControls = {
  /**
   * 设置进度条区域
   * @param {number} percent 百分比 0-100
   */
  setProgress(percent) {
    const inner = $('.progress').show().find('.inner');
    inner[0].clientHeight; // force reflow
    inner.css('width', `${percent}%`);
    inner.find('span').text(`${percent}%`);
  },
  /**
   * 设置上传按钮状态
   */
  setStatus() {
    const btn = $('.btn.control');
    const status = btn[0].dataset.status;
    switch (status) {
      case 'unchoose': // 未选择文件
        btn.hide();
        break;
      case 'choose': // 刚刚选择了文件
        btn.show();
        btn.text('开始上传');
        break;
      case 'uploading': // 上传中
        btn.show();
        btn.text('暂停');
        break;
      case 'pause': // 暂停中
        btn.show();
        btn.text('继续');
        break;
      case 'finish': // 已完成
        btn.hide();
        break;
    }
  },
  /**
   * 设置文件链接
   */
  setLink(link) {
    $('#link').show().find('a').prop('href', link).text(link);
  },
};

/**
 * 文件分片
 * @param {File} file
 * @returns
 */
async function splitFile(file) {
  return new Promise((resolve) => {
    // 分片尺寸（1M）
    const chunkSize = 1024 * 1024;
    // 分片数量
    const chunkCount = Math.ceil(file.size / chunkSize);
    // 当前chunk的下标
    let chunkIndex = 0;
    // 使用ArrayBuffer完成文件MD5编码
    const spark = new SparkMD5.ArrayBuffer();
    const fileReader = new FileReader(); // 文件读取器
    const chunks = []; // 分片信息数组
    // 读取一个分片后的回调
    fileReader.onload = function (e) {
      spark.append(e.target.result); // 分片数据追加到MD5编码器中
      // 当前分片单独的MD5
      const chunkMD5 = SparkMD5.ArrayBuffer.hash(e.target.result) + chunkIndex;
      chunkIndex++;
      chunks.push({
        id: chunkMD5,
        content: new Blob([e.target.result]),
      });
      if (chunkIndex < chunkCount) {
        loadNext(); // 继续读取下一个分片
      } else {
        // 读取完成
        const fileId = spark.end();
        resolve({
          fileId,
          ext: extname(file.name),
          chunks,
        });
      }
    };
    // 读取下一个分片
    function loadNext() {
      const start = chunkIndex * chunkSize,
        end = start + chunkSize >= file.size ? file.size : start + chunkSize;

      fileReader.readAsArrayBuffer(file.slice(start, end));
    }

    /**
     * 获取文件的后缀名
     * @param {string} filename 文件完整名称
     */
    function extname(filename) {
      const i = filename.lastIndexOf('.');
      if (i < 0) {
        return '';
      }
      return filename.substr(i);
    }

    loadNext();
  });
}

// 选择文件
$('.btn.choose').click(function () {
  $('#file').click();
});
let fileInfo;
let needs;

function setProgress() {
  const total = fileInfo.chunks.length;
  let percent = ((total - needs.length) / total) * 100;
  percent = Math.ceil(percent);
  domControls.setProgress(percent);
}
$('#file').change(async function () {
  $('.modal').show();
  fileInfo = await splitFile(this.files[0]);
  const resp = await fetch('http://localhost:8000/api/upload/handshake', {
    method: 'POST',
    headers: {
      'content-type': 'application/json',
    },
    body: JSON.stringify({
      fileId: fileInfo.fileId,
      ext: fileInfo.ext,
      chunkIds: fileInfo.chunks.map((it) => it.id),
    }),
  }).then((resp) => resp.json());
  $('.modal').hide();
  if (Array.isArray(resp.data)) {
    needs = resp.data;
    setProgress();
    $('.btn.control')[0].dataset.status = 'choose';
    domControls.setStatus();
  } else {
    needs = [];
    setProgress();
    $('.btn.control')[0].dataset.status = 'finish';
    domControls.setStatus();
    domControls.setLink(resp.data);
  }
});

$('.btn.control').click(function () {
  const status = this.dataset.status;
  switch (status) {
    case 'unchoose':
    case 'finish':
      return;
    case 'uploading':
      this.dataset.status = 'pause';
      domControls.setStatus();
      break;
    case 'choose':
    case 'pause':
      this.dataset.status = 'uploading';
      uploadPiece();
      domControls.setStatus();
      break;
  }
});

async function uploadPiece() {
  if (!needs) {
    return;
  }
  if (needs.length === 0) {
    // 上传完成
    setProgress();
    $('.btn.control')[0].dataset.status = 'finish';
    domControls.setStatus();
    domControls.setLink(
      `http://localhost:8000/upload/${fileInfo.fileId}${fileInfo.ext}`
    );
    return;
  }
  const status = $('.btn.control')[0].dataset.status;
  if (status !== 'uploading') {
    return;
  }
  const nextChunkId = needs[0];
  const file = fileInfo.chunks.find((it) => it.id === nextChunkId).content;
  const formData = new FormData();
  formData.append('file', file);
  formData.append('chunkId', nextChunkId);
  formData.append('fileId', fileInfo.fileId);
  const resp = await fetch('http://localhost:8000/api/upload', {
    method: 'POST',
    body: formData,
  }).then((resp) => resp.json());
  needs = resp.data;
  setProgress();
  uploadPiece();
}

打赏作者

您的打赏是我前进的动力

微信

支付宝

计算机网络🛜 TCP 协议

计算机网络🛜 域名和 DNS

中午好👏🏻，我是 ✍🏻 疯狂 codding 中...

粽子

这有关于前端开发的技术文档和你分享。

相信你可以在这里找到对你有用的知识和教程。

了解更多

1. 断点续传-下载
2. 断点续传-上传
1. 2.1. 服务端代码
2. 2.2. 客户端代码