ddosvn commited on
Commit
6397159
·
verified ·
1 Parent(s): 7c5da52

Turbo Parallel 8-Stream Chunked Upload for large files

Browse files
Files changed (2) hide show
  1. frontend/src/context/CloudContext.tsx +113 -1
  2. main.go +217 -0
frontend/src/context/CloudContext.tsx CHANGED
@@ -406,7 +406,7 @@ export function CloudProvider({ children }: { children: React.ReactNode }) {
406
  setUploadQueue(initialQueue);
407
 
408
  const totalBatchBytes = batch.reduce((acc, f) => acc + f.size, 0);
409
- if (batch.length > 1 && totalBatchBytes <= 512 * 1024 * 1024) {
410
  try {
411
  await uploadBatchSingleCommitXHR(batch, cleanFolder, (pct, speedStr, eta) => {
412
  setUploadQueue((prev) =>
@@ -514,11 +514,123 @@ export function CloudProvider({ children }: { children: React.ReactNode }) {
514
  });
515
  }
516
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
517
  function uploadSingleXHR(
518
  file: File,
519
  remotePath: string,
520
  onProgress: (pct: number, speedMBps: string, etaSec: number) => void
521
  ) {
 
 
 
522
  return new Promise((resolve, reject) => {
523
  const xhr = new XMLHttpRequest();
524
  const form = new FormData();
 
406
  setUploadQueue(initialQueue);
407
 
408
  const totalBatchBytes = batch.reduce((acc, f) => acc + f.size, 0);
409
+ if (batch.length > 1 && totalBatchBytes <= 16 * 1024 * 1024) {
410
  try {
411
  await uploadBatchSingleCommitXHR(batch, cleanFolder, (pct, speedStr, eta) => {
412
  setUploadQueue((prev) =>
 
514
  });
515
  }
516
 
517
+ async function uploadParallelChunked(
518
+ file: File,
519
+ remotePath: string,
520
+ onProgress: (pct: number, speedMBps: string, etaSec: number) => void
521
+ ) {
522
+ const authHeaders: Record<string, string> = { "Content-Type": "application/json" };
523
+ if (token) authHeaders["Authorization"] = `Bearer ${token}`;
524
+
525
+ const initRes = await apiFetch("/api/upload/init", {
526
+ method: "POST",
527
+ headers: authHeaders,
528
+ body: JSON.stringify({ path: remotePath, size: file.size }),
529
+ });
530
+ const initData = await initRes.json().catch(() => ({}));
531
+ if (!initRes.ok || !initData.upload_id) {
532
+ throw new Error(initData.detail || initData.error || `Init HTTP ${initRes.status}`);
533
+ }
534
+
535
+ const uploadId: string = initData.upload_id;
536
+ const chunkSize = 4 * 1024 * 1024; // 4 MiB per chunk
537
+ const totalChunks = Math.max(1, Math.ceil(file.size / chunkSize));
538
+ const chunkLoaded = new Float64Array(totalChunks);
539
+ const startedAt = performance.now();
540
+
541
+ const reportProgress = () => {
542
+ let loaded = 0;
543
+ for (let i = 0; i < totalChunks; i++) loaded += chunkLoaded[i];
544
+ const pct = file.size > 0 ? Math.min(99, Math.round((loaded / file.size) * 100)) : 99;
545
+ const elapsedSec = Math.max((performance.now() - startedAt) / 1000, 0.1);
546
+ const bps = loaded / elapsedSec;
547
+ const speedMBps = (bps / (1024 * 1024)).toFixed(1) + " MB/s";
548
+ const etaSec = bps > 0 ? Math.ceil(Math.max(0, file.size - loaded) / bps) : 0;
549
+ onProgress(pct, speedMBps, etaSec);
550
+ };
551
+
552
+ const sendChunkOnce = (chunkIndex: number, offset: number, blob: Blob): Promise<void> =>
553
+ new Promise((resolve, reject) => {
554
+ const xhr = new XMLHttpRequest();
555
+ xhr.upload.onprogress = (ev) => {
556
+ if (ev.lengthComputable) {
557
+ chunkLoaded[chunkIndex] = ev.loaded;
558
+ reportProgress();
559
+ }
560
+ };
561
+ xhr.onload = () => {
562
+ if (xhr.status >= 200 && xhr.status < 300) {
563
+ chunkLoaded[chunkIndex] = blob.size;
564
+ reportProgress();
565
+ resolve();
566
+ } else {
567
+ reject(new Error(`Chunk HTTP ${xhr.status}`));
568
+ }
569
+ };
570
+ xhr.onerror = () => reject(new Error("Chunk network error"));
571
+ xhr.open(
572
+ "POST",
573
+ `${getBackendOrigin()}/api/upload/chunk?id=${encodeURIComponent(uploadId)}&offset=${offset}`
574
+ );
575
+ xhr.setRequestHeader("Content-Type", "application/octet-stream");
576
+ if (token) xhr.setRequestHeader("Authorization", `Bearer ${token}`);
577
+ xhr.send(blob);
578
+ });
579
+
580
+ const sendChunkWithRetry = async (chunkIndex: number) => {
581
+ const offset = chunkIndex * chunkSize;
582
+ const end = Math.min(offset + chunkSize, file.size);
583
+ const blob = file.slice(offset, end);
584
+ let lastErr: any;
585
+ for (let attempt = 0; attempt < 3; attempt++) {
586
+ try {
587
+ await sendChunkOnce(chunkIndex, offset, blob);
588
+ return;
589
+ } catch (err) {
590
+ lastErr = err;
591
+ chunkLoaded[chunkIndex] = 0;
592
+ await new Promise((r) => setTimeout(r, 400 * (attempt + 1)));
593
+ }
594
+ }
595
+ throw lastErr;
596
+ };
597
+
598
+ const STREAM_CONCURRENCY = Math.min(8, totalChunks);
599
+ let nextChunk = 0;
600
+ const workers = Array.from({ length: STREAM_CONCURRENCY }, async () => {
601
+ while (nextChunk < totalChunks) {
602
+ const idx = nextChunk++;
603
+ await sendChunkWithRetry(idx);
604
+ }
605
+ });
606
+ await Promise.all(workers);
607
+
608
+ const elapsedSec = Math.max((performance.now() - startedAt) / 1000, 0.1);
609
+ const finalBps = file.size / elapsedSec;
610
+ const finalSpeed = (finalBps / (1024 * 1024)).toFixed(1) + " MB/s";
611
+ onProgress(100, finalSpeed, 0);
612
+
613
+ const compHeaders: Record<string, string> = {};
614
+ if (token) compHeaders["Authorization"] = `Bearer ${token}`;
615
+ const compRes = await apiFetch(`/api/upload/complete?id=${encodeURIComponent(uploadId)}`, {
616
+ method: "POST",
617
+ headers: compHeaders,
618
+ });
619
+ const compData = await compRes.json().catch(() => ({}));
620
+ if (!compRes.ok) {
621
+ throw new Error(compData.detail || compData.error || `Complete HTTP ${compRes.status}`);
622
+ }
623
+ return compData;
624
+ }
625
+
626
  function uploadSingleXHR(
627
  file: File,
628
  remotePath: string,
629
  onProgress: (pct: number, speedMBps: string, etaSec: number) => void
630
  ) {
631
+ if (file.size > 4 * 1024 * 1024) {
632
+ return uploadParallelChunked(file, remotePath, onProgress);
633
+ }
634
  return new Promise((resolve, reject) => {
635
  const xhr = new XMLHttpRequest();
636
  const form = new FormData();
main.go CHANGED
@@ -60,6 +60,30 @@ type App struct {
60
  listCacheTime time.Time
61
  pathAllocMu sync.Mutex
62
  reservedPaths map[string]bool
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  }
64
 
65
  func (a *App) invalidateListCache() {
@@ -177,6 +201,9 @@ func main() {
177
  mux.Handle("DELETE /api/admin/users", app.auth(http.HandlerFunc(app.handleAdminDeleteUser)))
178
  mux.Handle("POST /api/upload", app.auth(http.HandlerFunc(app.upload)))
179
  mux.Handle("POST /api/upload/batch", app.auth(http.HandlerFunc(app.uploadBatch)))
 
 
 
180
  mux.Handle("GET /api/files", app.auth(http.HandlerFunc(app.listFiles)))
181
  mux.Handle("PUT /api/files", app.auth(http.HandlerFunc(app.renameFile)))
182
  mux.Handle("DELETE /api/files", app.auth(http.HandlerFunc(app.deleteFile)))
@@ -619,6 +646,196 @@ func (a *App) uploadBatch(w http.ResponseWriter, r *http.Request) {
619
  })
620
  }
621
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
622
  func parseBucketLsLine(line string) (FileItem, bool) {
623
  trimmed := strings.TrimSpace(line)
624
  if trimmed == "" {
 
60
  listCacheTime time.Time
61
  pathAllocMu sync.Mutex
62
  reservedPaths map[string]bool
63
+ chunkSessions sync.Map // map[string]*chunkUploadSession
64
+ }
65
+
66
+ type chunkUploadSession struct {
67
+ id string
68
+ owner string
69
+ remotePath string
70
+ tmpPath string
71
+ file *os.File
72
+ totalSize int64
73
+ createdAt time.Time
74
+ mu sync.Mutex
75
+ closed bool
76
+ }
77
+
78
+ type offsetWriter struct {
79
+ f *os.File
80
+ off int64
81
+ }
82
+
83
+ func (ow *offsetWriter) Write(p []byte) (int, error) {
84
+ n, err := ow.f.WriteAt(p, ow.off)
85
+ ow.off += int64(n)
86
+ return n, err
87
  }
88
 
89
  func (a *App) invalidateListCache() {
 
201
  mux.Handle("DELETE /api/admin/users", app.auth(http.HandlerFunc(app.handleAdminDeleteUser)))
202
  mux.Handle("POST /api/upload", app.auth(http.HandlerFunc(app.upload)))
203
  mux.Handle("POST /api/upload/batch", app.auth(http.HandlerFunc(app.uploadBatch)))
204
+ mux.Handle("POST /api/upload/init", app.auth(http.HandlerFunc(app.uploadChunkInit)))
205
+ mux.Handle("POST /api/upload/chunk", app.auth(http.HandlerFunc(app.uploadChunkPart)))
206
+ mux.Handle("POST /api/upload/complete", app.auth(http.HandlerFunc(app.uploadChunkComplete)))
207
  mux.Handle("GET /api/files", app.auth(http.HandlerFunc(app.listFiles)))
208
  mux.Handle("PUT /api/files", app.auth(http.HandlerFunc(app.renameFile)))
209
  mux.Handle("DELETE /api/files", app.auth(http.HandlerFunc(app.deleteFile)))
 
646
  })
647
  }
648
 
649
+ func (a *App) cleanupStaleChunkSessions() {
650
+ now := time.Now()
651
+ a.chunkSessions.Range(func(key, value any) bool {
652
+ sess, ok := value.(*chunkUploadSession)
653
+ if !ok {
654
+ return true
655
+ }
656
+ if now.Sub(sess.createdAt) > 2*time.Hour {
657
+ sess.mu.Lock()
658
+ if !sess.closed {
659
+ sess.closed = true
660
+ _ = sess.file.Close()
661
+ _ = os.Remove(sess.tmpPath)
662
+ a.releaseReservedPath(sess.remotePath)
663
+ }
664
+ sess.mu.Unlock()
665
+ a.chunkSessions.Delete(key)
666
+ }
667
+ return true
668
+ })
669
+ }
670
+
671
+ func (a *App) uploadChunkInit(w http.ResponseWriter, r *http.Request) {
672
+ u := currentUser(r)
673
+ if u.Role != "admin" && !u.Permissions.CanUpload {
674
+ writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden", Detail: "Tài khoản của bạn không được cấp quyền tải tệp lên"})
675
+ return
676
+ }
677
+ a.cleanupStaleChunkSessions()
678
+
679
+ var req struct {
680
+ Path string `json:"path"`
681
+ Size int64 `json:"size"`
682
+ }
683
+ if err := json.NewDecoder(io.LimitReader(r.Body, 16384)).Decode(&req); err != nil {
684
+ writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_json", Detail: err.Error()})
685
+ return
686
+ }
687
+ if req.Size < 0 || req.Size > a.cfg.MaxUpload {
688
+ writeJSON(w, http.StatusRequestEntityTooLarge, APIError{Error: "file_too_large"})
689
+ return
690
+ }
691
+ clean, err := cleanRemotePath(req.Path)
692
+ if err != nil {
693
+ writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_path", Detail: err.Error()})
694
+ return
695
+ }
696
+ remotePath := a.resolveUniqueRemotePath(scopeUserRemotePath(u.Username, clean))
697
+ a.evictDownloadCache(remotePath)
698
+
699
+ tmp, err := os.CreateTemp(a.cfg.TempDir, "hf-chunked-*")
700
+ if err != nil {
701
+ a.releaseReservedPath(remotePath)
702
+ writeJSON(w, http.StatusInternalServerError, APIError{Error: "temp_file_error", Detail: err.Error()})
703
+ return
704
+ }
705
+ if req.Size > 0 {
706
+ if err := tmp.Truncate(req.Size); err != nil {
707
+ _ = tmp.Close()
708
+ _ = os.Remove(tmp.Name())
709
+ a.releaseReservedPath(remotePath)
710
+ writeJSON(w, http.StatusInternalServerError, APIError{Error: "temp_alloc_error", Detail: err.Error()})
711
+ return
712
+ }
713
+ }
714
+
715
+ uploadID := randomHex(16)
716
+ sess := &chunkUploadSession{
717
+ id: uploadID,
718
+ owner: u.Username,
719
+ remotePath: remotePath,
720
+ tmpPath: tmp.Name(),
721
+ file: tmp,
722
+ totalSize: req.Size,
723
+ createdAt: time.Now(),
724
+ }
725
+ a.chunkSessions.Store(uploadID, sess)
726
+
727
+ writeJSON(w, http.StatusOK, map[string]any{
728
+ "ok": true,
729
+ "upload_id": uploadID,
730
+ "path": remotePath,
731
+ "chunk_size": 4 << 20, // 4 MiB default chunk
732
+ })
733
+ }
734
+
735
+ func (a *App) uploadChunkPart(w http.ResponseWriter, r *http.Request) {
736
+ u := currentUser(r)
737
+ uploadID := strings.TrimSpace(r.URL.Query().Get("id"))
738
+ offsetStr := strings.TrimSpace(r.URL.Query().Get("offset"))
739
+ offset, err := strconv.ParseInt(offsetStr, 10, 64)
740
+ if uploadID == "" || err != nil || offset < 0 {
741
+ writeJSON(w, http.StatusBadRequest, APIError{Error: "invalid_params", Detail: "missing id or valid offset"})
742
+ return
743
+ }
744
+ val, ok := a.chunkSessions.Load(uploadID)
745
+ if !ok {
746
+ writeJSON(w, http.StatusNotFound, APIError{Error: "session_not_found", Detail: "Phiên tải lên không tồn tại hoặc đã hết hạn"})
747
+ return
748
+ }
749
+ sess := val.(*chunkUploadSession)
750
+ if sess.owner != u.Username && u.Role != "admin" {
751
+ writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden"})
752
+ return
753
+ }
754
+
755
+ buf := make([]byte, 1<<20) // 1 MiB copy buffer per parallel stream
756
+ ow := &offsetWriter{f: sess.file, off: offset}
757
+ written, copyErr := io.CopyBuffer(ow, io.LimitReader(r.Body, 64<<20), buf)
758
+ if copyErr != nil {
759
+ writeJSON(w, http.StatusBadRequest, APIError{Error: "chunk_write_error", Detail: copyErr.Error()})
760
+ return
761
+ }
762
+
763
+ writeJSON(w, http.StatusOK, map[string]any{
764
+ "ok": true,
765
+ "written": written,
766
+ "offset": offset,
767
+ })
768
+ }
769
+
770
+ func (a *App) uploadChunkComplete(w http.ResponseWriter, r *http.Request) {
771
+ start := time.Now()
772
+ u := currentUser(r)
773
+ uploadID := strings.TrimSpace(r.URL.Query().Get("id"))
774
+ if uploadID == "" {
775
+ writeJSON(w, http.StatusBadRequest, APIError{Error: "missing_id"})
776
+ return
777
+ }
778
+ val, ok := a.chunkSessions.LoadAndDelete(uploadID)
779
+ if !ok {
780
+ writeJSON(w, http.StatusNotFound, APIError{Error: "session_not_found"})
781
+ return
782
+ }
783
+ sess := val.(*chunkUploadSession)
784
+ if sess.owner != u.Username && u.Role != "admin" {
785
+ writeJSON(w, http.StatusForbidden, APIError{Error: "forbidden"})
786
+ return
787
+ }
788
+
789
+ sess.mu.Lock()
790
+ sess.closed = true
791
+ _ = sess.file.Close()
792
+ sess.mu.Unlock()
793
+
794
+ defer func() {
795
+ _ = os.Remove(sess.tmpPath)
796
+ }()
797
+
798
+ if a.pool != nil {
799
+ _, err := a.pool.Call(map[string]any{
800
+ "op": "upload",
801
+ "bucket": bucketID,
802
+ "files": []map[string]string{
803
+ {"local": sess.tmpPath, "remote": sess.remotePath},
804
+ },
805
+ })
806
+ if err != nil {
807
+ a.releaseReservedPath(sess.remotePath)
808
+ writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_upload_failed", Detail: err.Error()})
809
+ return
810
+ }
811
+ } else {
812
+ dest := "hf://buckets/" + bucketID + "/" + sess.remotePath
813
+ if out, err := a.runHF(r.Context(), "buckets", "cp", sess.tmpPath, dest); err != nil {
814
+ a.releaseReservedPath(sess.remotePath)
815
+ writeJSON(w, http.StatusBadGateway, APIError{Error: "huggingface_upload_failed", Detail: out})
816
+ return
817
+ }
818
+ }
819
+
820
+ if a.owners != nil {
821
+ a.owners.SetOwner(sess.remotePath, sess.owner)
822
+ }
823
+ if a.mongo != nil {
824
+ a.mongo.UpsertFileMeta(sess.remotePath, sess.owner, sess.totalSize)
825
+ }
826
+ a.invalidateListCache()
827
+
828
+ writeJSON(w, http.StatusOK, map[string]any{
829
+ "ok": true,
830
+ "bucket": bucketID,
831
+ "path": sess.remotePath,
832
+ "owner": sess.owner,
833
+ "size": sess.totalSize,
834
+ "duration_ms": time.Since(start).Milliseconds(),
835
+ "engine": "xet-parallel-chunked",
836
+ })
837
+ }
838
+
839
  func parseBucketLsLine(line string) (FileItem, bool) {
840
  trimmed := strings.TrimSpace(line)
841
  if trimmed == "" {