transmux: use smaller chunks when concatenating long list of files

video/transmux.go

@@ -18,6 +18,7 @@ import (
 
 const (
 	Mp4DurationLimit = 21600 //MP4s will be generated only for first 6 hours
+	MaxArgLimit      = 250
 )
 
 func MuxTStoMP4(tsInputFile, mp4OutputFile string) ([]string, error) {
@@ -141,6 +142,29 @@ func ConcatTS(tsFileName string, segmentsList *TSegmentList, sourceMediaPlaylist
 				break
 			}
 		}
+		// If the argument list of files gets too long, linux might complain about exceeding
+		// MAX_ARG limit and ffmpeg (or any other command) using the long list will fail to run.
+		// So we split into chunked files then concat it one final time to get the final file.
+		if len(segmentFilenames) > MaxArgLimit {
+			chunks := ConcatChunkedFiles(segmentFilenames, MaxArgLimit)
+
+			var chunkFiles []string
+			for idx, chunk := range chunks {
+				concatArg := "concat:" + strings.Join(chunk, "|")
+				chunkFilename := fileBaseWithoutExt + "_" + "chunk" + strconv.Itoa(idx) + ".ts"
+				chunkFiles = append(chunkFiles, chunkFilename)
+				err := concatFiles(concatArg, chunkFilename)
+				if err != nil {
+					return totalBytes, fmt.Errorf("failed to file-concat a chunk (#%d)into a ts file: %w", idx, err)
+				}
+			}
+			if len(chunkFiles) == 0 {
+				return totalBytes, fmt.Errorf("failed to generate chunks to concat")
+			}
+			// override with the chunkFilenames instead
+			segmentFilenames = chunkFiles
+
+		}
 		concatArg := "concat:" + strings.Join(segmentFilenames, "|")
 
 		// Use file-based concatenation by reading segment files in text file
@@ -150,7 +174,6 @@ func ConcatTS(tsFileName string, segmentsList *TSegmentList, sourceMediaPlaylist
 		}
 
 		return totalBytes, nil
-
 	} else {
 		// Create a text file containing filenames of the segments
 		segmentListTxtFileName := fileBaseWithoutExt + ".txt"
@@ -255,3 +278,18 @@ func concatFiles(segmentList, outputTsFileName string) error {
 	}
 	return nil
 }
+
+// ConcatChunkedFiles splits the segmentFilenames into smaller chunks based on the maxLength value,
+// where maxLength is the maximum number of filenames per chunk.
+func ConcatChunkedFiles(filenames []string, maxLength int) [][]string {
+	var chunks [][]string
+	for maxLength > 0 && len(filenames) > 0 {
+		if len(filenames) <= maxLength {
+			chunks = append(chunks, filenames)
+			break
+		}
+		chunks = append(chunks, filenames[:maxLength])
+		filenames = filenames[maxLength:]
+	}
+	return chunks
+}

video/transmux_test.go

@@ -5,6 +5,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"os"
 	"path/filepath"
+	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 )
@@ -213,6 +215,53 @@ func TestItConcatsStreamsOnlyUptoMP4DurationLimit(t *testing.T) {
 	require.Equal(t, int64(406268), totalBytesW)
 }
 
+func TestConcatChunkedFiles(t *testing.T) {
+	filenames := make([]string, 10)
+	for i := range filenames {
+		filenames[i] = "file" + strconv.Itoa(i+1)
+	}
+
+	testCases := []struct {
+		name       string
+		maxLength  int
+		wantChunks [][]string
+	}{
+		{
+			name:      "MaxLengthLessThanLength",
+			maxLength: 3,
+			wantChunks: [][]string{
+				{"file1", "file2", "file3"},
+				{"file4", "file5", "file6"},
+				{"file7", "file8", "file9"},
+				{"file10"},
+			},
+		},
+		{
+			name:      "MaxLengthEqualToLength",
+			maxLength: 10,
+			wantChunks: [][]string{
+				{"file1", "file2", "file3", "file4", "file5", "file6", "file7", "file8", "file9", "file10"},
+			},
+		},
+		{
+			name:      "MaxLengthGreaterThanLength",
+			maxLength: 15,
+			wantChunks: [][]string{
+				{"file1", "file2", "file3", "file4", "file5", "file6", "file7", "file8", "file9", "file10"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gotChunks := ConcatChunkedFiles(filenames, tc.maxLength)
+			if !reflect.DeepEqual(gotChunks, tc.wantChunks) {
+				t.Errorf("ConcatChunkedFiles(%v, %d) = %v, want %v", filenames, tc.maxLength, gotChunks, tc.wantChunks)
+			}
+		})
+	}
+}
+
 func populateRenditionSegmentList() *TRenditionList {
 	segmentFiles := []string{"../test/fixtures/seg-0.ts", "../test/fixtures/seg-1.ts", "../test/fixtures/seg-2.ts"}