大家好，欢迎来到IT知识分享网。

日常如果是上传一些小文件，在程序实现中，我们都是直接上传，一般都没什么问题。如果针对大文件上传的业务中，就会面临着：

1、网速问题，导致文件上传超时，而导致失败。

2、效率问题，上传大文件等待时间过长，如果是需要上传多个，就会更慢。

3、体验问题，用户无法预知上传还需花费的时间，系统没有及时反馈，用户无法判断文件是否还在上传，还是断开。

这时候就需要采用分布式文件上传系统。

项目简介

这是一个基于.Net Core构建的简单、跨平台分布式文件上传系统，支持分块上传、多个项目同时上传、接口权限控制采用JWT机制。

技术架构

1、跨平台：这是基于.Net Core开发的系统，可以部署在Docker, Windows, Linux, Mac。

2、.Net 2.1 + Jwt + simple-uploader

项目结构

项目分为分块上传与一般上传Demo，Web、控制台上传Demo。ufs项目是分布式文件上传的统一接口，ufs会根据配置把上传的文件发到ufs.node节点，ufs.node会把上传成功路径返回给ufs并存储，用户访问的时候，ufs会访问对应节点返回资源。

UploadServer为一般文件上传接口，UploadServer.FrontEndDemo为Web上传文件Demo。

使用

1、配置

配置允许上传域名、服务接口地址、允许的文件格式、文件大小、存储路径等信息。

{

  "AllowedHosts": "*",

  "urls": "http://localhost:6001",

  "uploadServer": {

    "rootUrl": "http://localhost:6001",

    "entryPoint1": "/upload",

    "entryPoint2": "/chunkUpload",

    "virtualPath": "",

    "physicalPath": "/Users/loogn/Desktop/uploader",

    "appendMimes": ".htm3:text/html;",

    "responseCache": 604800,

    "jwtSecret": "1234561234",

    "limitSize": "20mb",

    "allowExts": ".txt;.jpg;.jpeg;.png;.doc;.docx;.xls;.xlsx;.ppt;.pptx;.pdf",

    "apps": {

      "default": {

        "allowOrigins": "",

        "enableThumbnail": true,

        "limitExts": ".exe;",

        "thumbnailExts": ".jpg;.jpeg;.png;"

      },

      "app1": {

        "allowOrigins": "*"

      }

    }

  }

}

2、前端

一般上传代码

$("#file1").change(function () { $.ajaxFileUpload({ fileElementId: 'file1', url: 'http://localhost:6001/upload', dataType: 'text', // success: function (data) { console.log("上传成功：", data); }, data: { "jwt": jwt } }); });

分块上传

var uploader = new Uploader({ target: 'http://localhost:6001/chunkupload', headers: {jwt: jwt} }); uploader.assignBrowse(document.getElementById('browseButton')); //uploader.assignBrowse(document.getElementById('folderButton'), true); // // 文件添加 单个文件 uploader.on('fileAdded', function (file, event) { console.log("fileAdded:", file, event) }); // 单个文件上传成功 uploader.on('fileSuccess', function (rootFile, file, message) { console.log("fileSuccess：", rootFile, file, message) }); // 根下的单个文件（文件夹）上传完成 uploader.on('fileComplete', function (rootFile) { console.log("fileComplete:", rootFile) }); // 某个文件上传失败了 uploader.on('fileError', function (rootFile, file, message) { console.log("fileError:", rootFile, file, message) });

3、后端

一般上传

 public async Task InvokeAsync(HttpContext context, RequestDelegate next) { context.Response.Headers.Add("Access-Control-Allow-Origin", "*"); context.Response.Headers.Add("Access-Control-Allow-Headers", "content-type,jwt,origin"); if (context.Request.Method.Equals(HttpMethods.Options, StringComparison.OrdinalIgnoreCase)) { context.Response.StatusCode = (int) HttpStatusCode.OK; } else if (context.Request.Method.Equals(HttpMethods.Post, StringComparison.OrdinalIgnoreCase)) { //验证jwt string token = null; if (context.Request.Headers.TryGetValue("jwt", out StringValues jwt)) { token = jwt.ToString(); } else if (context.Request.Form.TryGetValue("jwt", out jwt)) { token = jwt.ToString(); } else { await context.Response.WriteAsync(new UploadResult() { msg = "No JWT in the header and form" }.toJson()); return; } try { var payload = new JwtBuilder().WithSecret(_config.JWTSecret).MustVerifySignature() .Decode<JwtPayload>(token); var msg = payload.validate(); if (msg != null) { await context.Response.WriteAsync(new UploadResult() { msg = msg }.toJson()); return; } //特定的配置 var appConfig = _config.GetAppConfig(payload.app); //跨域 context.Request.Headers.TryGetValue("Origin", out var origins); var origin = origins.ToString(); if (!string.IsNullOrEmpty(origin) && appConfig.IsAllowOrigin(origin)) { context.Response.Headers.Add("Access-Control-Allow-Origin", origin); } //获取上传的文件 var file = context.Request.Form.Files.FirstOrDefault(); if (file == null || file.Length == 0) { await context.Response.WriteAsync(new UploadResult() { msg = "There is no file data" }.toJson()); return; } //大小验证 if (file.Length > (payload.GetByteSize() ?? _config.GetByteSize())) { await context.Response.WriteAsync(new UploadResult() { msg = "The file is too big" }.toJson()); return; } //后缀验证 var ext = Path.GetExtension(file.FileName); if (!(payload.exts + _config.AllowExts).Contains(ext, StringComparison.OrdinalIgnoreCase) || appConfig.LimitExts.Contains(ext, StringComparison.OrdinalIgnoreCase)) { await context.Response.WriteAsync(new UploadResult() { msg = "File extension is not allowed" }.toJson()); return; } //上传逻辑 var now = DateTime.Now; var yy = now.ToString("yyyy"); var mm = now.ToString("MM"); var dd = now.ToString("dd"); var fileName = Guid.NewGuid().ToString("n") + ext; var folder = Path.Combine(_config.PhysicalPath, payload.app, yy, mm, dd); if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } var filePath = Path.Combine(folder, fileName); using (var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write)) { file.CopyTo(fileStream); fileStream.Flush(true); } var fileUrl = _config.RootUrl + "/" + payload.app + "/" + yy + "/" + mm + "/" + dd + "/" + fileName; await context.Response.WriteAsync(new UploadResult() { ok = true, url = fileUrl }.toJson()); } catch (TokenExpiredException) { await context.Response.WriteAsync(new UploadResult() { msg = "Token has expired" }.toJson()); } catch (SignatureVerificationException) { await context.Response.WriteAsync(new UploadResult() { msg = "Token has invalid signature" }.toJson()); } } else { await context.Response.WriteAsync(new UploadResult() { msg = #34;Request method '{context.Request.Method}' is not supported" }.toJson()); } }

分块上传

public async Task InvokeAsync(HttpContext context, RequestDelegate next) { context.Response.Headers.Add("Access-Control-Allow-Origin", "*"); context.Response.Headers.Add("Access-Control-Allow-Headers", "content-type,jwt,origin"); if (context.Request.Method.Equals(HttpMethods.Options, StringComparison.OrdinalIgnoreCase)) { context.Response.StatusCode = (int)HttpStatusCode.OK; } else if (context.Request.Method.Equals(HttpMethods.Get, StringComparison.OrdinalIgnoreCase)) { //简单实现 context.Request.Query.TryGetValue("chunkNumber", out var chunkNumbers); int.TryParse(chunkNumbers.ToString(), out var chunkNumber); context.Request.Query.TryGetValue("identifier", out var identifiers); if (chunkNumber == 0 || string.IsNullOrEmpty(identifiers)) { context.Response.StatusCode = 204; } else { var chunkFilename = getChunkFilename(_config.PhysicalPath, chunkNumber, identifiers); if (File.Exists(chunkFilename)) { await context.Response.WriteAsync("found"); } else { context.Response.StatusCode = 204; } } } else if (context.Request.Method.Equals(HttpMethods.Post, StringComparison.OrdinalIgnoreCase)) { //验证jwt string token = null; if (context.Request.Headers.TryGetValue("jwt", out StringValues jwt)) { token = jwt.ToString(); } else if (context.Request.Form.TryGetValue("jwt", out jwt)) { token = jwt.ToString(); } else { await context.Response.WriteAsync(new UploadResult() { msg = "No JWT in the header and form" }.toJson()); return; } try { var payload = new JwtBuilder().WithSecret(_config.JWTSecret).MustVerifySignature() .Decode<JwtPayload>(token); var msg = payload.validate(); if (msg != null) { await context.Response.WriteAsync(new UploadResult() { msg = msg }.toJson()); return; } //特定的配置 var appConfig = _config.GetAppConfig(payload.app); //跨域 context.Request.Headers.TryGetValue("Origin", out var origins); var origin = origins.ToString(); if (!string.IsNullOrEmpty(origin) && appConfig.IsAllowOrigin(origin)) { context.Response.Headers.Add("Access-Control-Allow-Origin", origin); } //获取上传的文件分片 var file = context.Request.Form.Files.FirstOrDefault(); if (file == null || file.Length == 0) { await context.Response.WriteAsync(new UploadResult() { msg = "There is no file data" }.toJson()); return; } //后缀验证 var ext = Path.GetExtension(file.FileName); if (!(payload.exts + _config.AllowExts).Contains(ext, StringComparison.OrdinalIgnoreCase) || appConfig.LimitExts.Contains(ext, StringComparison.OrdinalIgnoreCase)) { await context.Response.WriteAsync(new UploadResult() { msg = "File extension is not allowed" }.toJson()); return; } //获取参数 getParams(context, out var chunkNumber, out var chunkSize, out var totalSize, out string identifier, out string filename, out int totalChunks); //验证参数 var validMsg = validateRequest(chunkNumber, chunkSize, totalSize, identifier, filename, file.Length, totalChunks, payload.GetByteSize() ?? _config.GetByteSize()); if (validMsg != null) { await context.Response.WriteAsync(new UploadResult() { msg = validMsg }.toJson()); return; } else { var chunkFilename = getChunkFilename(_config.PhysicalPath, chunkNumber, identifier); try { using (var fileStream = File.OpenWrite(chunkFilename)) { var stream = file.OpenReadStream(); stream.CopyTo(fileStream); fileStream.Flush(true); countDict.AddOrUpdate(identifier, 1, (key, oldValue) => oldValue + 1); } if (chunkNumber == totalChunks) { //验证块的完整性 while (true) { if (countDict.GetValueOrDefault(identifier) < totalChunks) { await Task.Delay(TimeSpan.FromMilliseconds(500)); } else { countDict.Remove(identifier, out _); break; } } //merge file; string[] chunkFiles = Directory.GetFiles( Path.Combine(_config.PhysicalPath, temporaryFolder), "uploader-" + identifier + ".*", SearchOption.TopDirectoryOnly); var fileUrl = await MergeChunkFiles(payload, ext, chunkFiles); await context.Response.WriteAsync(new UploadResult() { ok = true, url = fileUrl }.toJson()); } else { await context.Response.WriteAsync("partly_done"); return; } } catch (Exception exp) { await context.Response.WriteAsync(new UploadResult() { msg = exp.Message }.toJson()); return; } } } catch (TokenExpiredException) { await context.Response.WriteAsync(new UploadResult() { msg = "Token has expired" }.toJson()); } catch (SignatureVerificationException) { await context.Response.WriteAsync(new UploadResult() { msg = "Token has invalid signature" }.toJson()); } } else { context.Response.StatusCode = (int)HttpStatusCode.MethodNotAllowed; await context.Response.WriteAsync(#34;Request method '{context.Request.Method}' is not supported"); } }

4、上传结果

上传成功

{"ok":true,"msg":null,"url":"http://localhost:6001/test/2019/06/17/abcd.jpg"} 

上传失败

{"ok":false,"msg":"The file is too big","url":null}

源代码地址

私信回复：1049