FS-10615 - Allow upload files bigger than 50GB (#511)

* Increase the chunks size to upload the max file size

* Chunk size and chunk count calculation updated for files greater than 60GB for default case and 80GB for intelligent ingestion

* Default values re-stored

---------

Co-authored-by: Yaminim <yaminim@celestialsys.com>

Author: prem-celestial

src/lib/api/upload/file.ts

@@ -52,6 +52,11 @@ export interface FileChunk extends FilePart {
   offset: number; // offset for chunk - from part start
 }
 
+export interface PartSize {
+  partsCount: number,
+  chunkSize: number
+}
+
 /**
  * File representation to unify file object in nodejs and browser
  *
@@ -164,14 +169,33 @@ export class File {
   }
 
   /**
-   * Returns number of parts according to part size
-   *
+   * Returns number of parts and part size according to max limit
    * @param {number} size - part size in bytes
-   * @returns {number}
+   * @returns {PartSize} 
    * @memberof File
    */
-  public getPartsCount (size: number): number {
-    return Math.ceil(this._file.size / size);
+  public getPartsCount (size: number, intelligentChunk: boolean): PartSize {
+    const DEFAULT_FILE_SIZE_LIMIT = 60 * 1024 * 1024 * 1024
+    const INTELLIGENT_FILE_SIZE_LIMIT = 80 * 1024 * 1024 * 1024
+    const FILE_SIZE_LIMIT = intelligentChunk ? INTELLIGENT_FILE_SIZE_LIMIT : DEFAULT_FILE_SIZE_LIMIT;
+    const MAX_S3_CHUNKS_ALLOWED = 10000;
+    
+    // When file size is greater than 60GB, chunk size is calculated dynamically
+    // Chunk count is set to the max number of chunks allowed over s3
+    if(this._file.size > FILE_SIZE_LIMIT) {
+      const dynamicPartSize = Math.ceil(this._file.size / MAX_S3_CHUNKS_ALLOWED); // size is set in bytes
+
+      return {
+        partsCount: Math.ceil(this._file.size / dynamicPartSize),
+        chunkSize: dynamicPartSize
+      }
+
+    }
+
+    return {
+      partsCount: Math.ceil(this._file.size / size),
+      chunkSize: size
+    }
   }
 
   /**

src/lib/api/upload/uploaders/s3.ts

@@ -254,19 +254,21 @@ export class S3Uploader extends UploaderAbstract {
    */
   private prepareParts(id: string): Promise<void> {
     const file = this.getPayloadById(id).file;
+    let intelligentChunk = false;
 
     // for intelligent or fallback mode we cant overwrite part size - requires 8MB
     if ([UploadMode.INTELLIGENT, UploadMode.FALLBACK].indexOf(this.uploadMode) > -1) {
       this.partSize = INTELLIGENT_CHUNK_SIZE;
+      intelligentChunk = true;
     }
 
-    const partsCount = file.getPartsCount(this.partSize);
+    const { partsCount, chunkSize } = file.getPartsCount(this.partSize, intelligentChunk);
 
     const parts = [];
 
     for (let i = 0; i < partsCount; i++) {
       parts[i] = {
-        ...file.getPartMetadata(i, this.partSize),
+        ...file.getPartMetadata(i, chunkSize),
         offset: 0,
       };
     }