From 1f43b3b5c7d92cf043610d7d25850266ebdf7ae0 Mon Sep 17 00:00:00 2001 From: sam-pich Date: Tue, 19 Nov 2024 16:46:34 -0500 Subject: [PATCH] FFMPEG complete, upload complete, todo list at top of file --- transfer/tshweb/go.mod | 7 + transfer/tshweb/go.sum | 14 ++ transfer/tshweb/transfer.go | 313 +++++++++++++++++++++++++++++++++--- 3 files changed, 314 insertions(+), 20 deletions(-) create mode 100644 transfer/tshweb/go.mod create mode 100644 transfer/tshweb/go.sum diff --git a/transfer/tshweb/go.mod b/transfer/tshweb/go.mod new file mode 100644 index 0000000..f334003 --- /dev/null +++ b/transfer/tshweb/go.mod @@ -0,0 +1,7 @@ +module filetransfer + +go 1.23.1 + +require github.com/aws/aws-sdk-go v1.55.5 + +require github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/transfer/tshweb/go.sum b/transfer/tshweb/go.sum new file mode 100644 index 0000000..d323266 --- /dev/null +++ b/transfer/tshweb/go.sum @@ -0,0 +1,14 @@ +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/transfer/tshweb/transfer.go b/transfer/tshweb/transfer.go index d26ff47..ca28b6a 100644 --- a/transfer/tshweb/transfer.go +++ b/transfer/tshweb/transfer.go @@ -1,22 +1,122 @@ package main import ( + "bytes" "fmt" "io" "net/http" + "os" "os/exec" + "path/filepath" + "strconv" "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" ) +// NEED TO WORK ON ASYNC (once og fileuploaded to server, treat it as if its regular download, but run the ffmpeg and r2 upload in background) +// Make display page for HTML during file upload +// Make youtube esque looking page to watch the bucket files + +// uploadToR2 uploads all files in the HLS output directory to R2 +func uploadToR2(r2Client *s3.S3, outputDir, bucketName string) error { + // Walk through the directory and upload each file + err := filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Skip directories (not rly needed in our case) + if info.IsDir() { + return nil + } + + // Open the file + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("error opening file %s: %v", path, err) + } + defer file.Close() + + // Determine the key (object name) in the bucket + // This preserves the directory structure relative to the base output directory + relativePath, err := filepath.Rel(outputDir, path) + if err != nil { + return err + } + + // Normalize path separators for R2 + key := strings.ReplaceAll(relativePath, "\\", "/") + + // Prepare the upload input, should upload to thirdculture bucket, with folder/key, and the files inside + uploadInput := &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: file, + } + + // Upload the file + _, err = r2Client.PutObject(uploadInput) + if err != nil { + return fmt.Errorf("error uploading file %s to R2: %v", path, err) + } + + fmt.Printf("Uploaded: %s\n", key) + return nil + }) + + return err +} + +// NewR2Client creates a new R2 client using AWS SDK +func NewR2Client(accountId, accessKeyId, secretAccessKey string) (*s3.S3, error) { + // Cloudflare R2 endpoint follows this format + r2Endpoint := fmt.Sprintf("https://%s.r2.cloudflarestorage.com", accountId) + + // Create AWS session with R2 credentials + sess, err := session.NewSession(&aws.Config{ + Endpoint: aws.String(r2Endpoint), + Region: aws.String("auto"), // Cloudflare uses 'auto' + Credentials: credentials.NewStaticCredentials(accessKeyId, secretAccessKey, ""), + }) + if err != nil { + return nil, fmt.Errorf("error creating AWS session: %v", err) + } + + // Create S3 client (used for R2) + return s3.New(sess), nil +} + +// In your existing uploadHandler function, add this after HLS conversion: +func uploadHLSToR2(r2Client *s3.S3, outputDir string) error { + // Upload to R2 bucket + err := uploadToR2(r2Client, outputDir, "thirdculture") + if err != nil { + return fmt.Errorf("error uploading HLS files to R2: %v", err) + } + // After successful upload, remove the local HLS directory + err = os.RemoveAll(outputDir) + if err != nil { + return fmt.Errorf("error deleting local HLS directory %s: %v", outputDir, err) + } + + fmt.Printf("Successfully uploaded and deleted local HLS directory: %s\n", outputDir) + return nil +} + func uploadHandler(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPost { + // Parse the form err := r.ParseMultipartForm(15 * 1024 * 1024 * 1024) // limit 15 gb if err != nil { http.Error(w, "Unable to parse form", http.StatusBadRequest) return } - file, _, err := r.FormFile("file") + file, fileHeader, err := r.FormFile("file") if err != nil { http.Error(w, "Unable to retrieve file", http.StatusBadRequest) return @@ -29,6 +129,18 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, "Error creating request", http.StatusInternalServerError) return } + // Filetype + // Buffer needed to make it faster, so we only process one bit of the file instead of all + buffer := make([]byte, 512) + _, err = file.Read(buffer) + if err != nil && err != io.EOF { + http.Error(w, "Error reading the file", http.StatusInternalServerError) + return + } + + fileType := http.DetectContentType(buffer) + file.Seek(0, 0) + client := &http.Client{} resp, err := client.Do(req) if err != nil { @@ -46,15 +158,180 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { urlParts := strings.Split(string(responseURL), "/") uniqueID := urlParts[len(urlParts)-1] - // Convert file to HLS. Work on error catching and flags tomorrow - // IF VIDEO=DO THIS. IF NOT CONTINUE! - // ALSO CHANGE THE CODE. IF VIDEO, REDIRECT TO S3 BUCKET - // CURRENTLY FILE IS STANDARD UPLOAD TO SERVER - cmd := exec.Command("ffmpeg") + // If its a video, proceed with encoding and seperate uploads + if strings.HasPrefix(fileType, "video/") { + // Convert file to HLS. Work on error catching and flags tomorrow + // IF VIDEO=DO THIS. IF NOT CONTINUE! + // ALSO CHANGE THE CODE. IF VIDEO, REDIRECT TO S3 BUCKET + // CURRENTLY FILE IS STANDARD UPLOAD TO SERVER + // + // Init r2 client + // API KEYS PUSHED TO GIT IS BAD, I KNOW. But this stuff isnt public and were using your git so idt anyone could crawl + // find my api keys and abuse this + // Plus this way you can access it too + r2Client, err := NewR2Client( + "58916dd30d55baca3a55ad78f0883c52", // Account + "94785bc0756ccedfe3634a7c4c7965f8", // Access + "d5666b49c672c8584bc3a393701a938ddd8412663d5ea470e2c372179a86245e", // Secret Access + ) + if err != nil { + http.Error(w, "Failed to initialize R2 client", http.StatusInternalServerError) + return + } - // HTML - w.Header().Set("Content-Type", "text/html") - fmt.Fprint(w, ` + // File vars for ffmpeg + inputDir := "../uploads" + err = os.MkdirAll(inputDir, os.ModePerm) + fileName := fileHeader.Filename + noExtensionFilename := strings.TrimSuffix(filepath.Base(fileName), filepath.Ext(fileName)) + + //Create output basedir + var outputBaseDir string + var outputDir string + if len(os.Args) >= 2 { + outputBaseDir = os.Args[1] + // Create dir if it doesnt exist + err := os.MkdirAll(outputBaseDir, os.ModePerm) + if err != nil { + fmt.Printf("Error creating directory: %v\n", err) + os.Exit(1) + } else { + // Default to $PWD + wd, err := os.Getwd() + if err != nil { + fmt.Printf("Error getting current directory: %v\n", err) + os.Exit(1) + } + outputBaseDir = wd + } + } + + // Set specific output diir + outputDir = outputBaseDir + "/" + noExtensionFilename + + // Check if the output directory exists + _, err = os.Stat(outputDir) + if err == nil { + // If directory exists, remove it + fmt.Printf("Output directory '%s' already exists. Removing it to avoid unused files.\n", outputDir) + err := os.RemoveAll(outputDir) // Removes the directory and its contents + if err != nil { + fmt.Printf("Error removing directory: %v\n", err) + return + } + } + + // FFPROBE to get frame rate + cmdFFPROBE := exec.Command("ffprobe", "-v", "0", "-of", "default=noprint_wrappers=1:nokey=1", + "-select_streams", "v:0", "-show_entries", "stream=avg_frame_rate", fileName) + + outputFrame, err := cmdFFPROBE.Output() + if err != nil { + fmt.Printf("Error running ffprobe: %v\n", err) + return + } + + frameRateStr := strings.TrimSpace(string(outputFrame)) + // Split the frame rate (which might be like "30/1" or "29.97") + parts := strings.Split(frameRateStr, "/") + var frameRateFloat float64 + + if len(parts) > 1 { + // Handle fraction (e.g., "30/1") + numerator, _ := strconv.ParseFloat(parts[0], 64) + denominator, _ := strconv.ParseFloat(parts[1], 64) + frameRateFloat = numerator / denominator + } else { + // Handle direct float (e.g., "29.97") + frameRateFloat, err = strconv.ParseFloat(frameRateStr, 64) + if err != nil { + fmt.Printf("Framerate parsing error: %v\n", err) + return + } + } + + // Convert to int for GOP calculation + GOP_SIZE := int(frameRateFloat * 4) + gopSizeStr := strconv.Itoa(GOP_SIZE) + + // resolution, bitrate, output slices/lists + resolutions := []string{"1280x720", "1920x1080", "3840x2160"} + bitrates := []string{"1200k", "2500k", "8000k"} + outputs := []string{"720p", "1080p", "2160p"} + playlist := []string{} + + for i := range resolutions { + res := resolutions[i] + bitrate := bitrates[i] + outputName := outputs[i] + playlistName := outputName + ".m3u8" + playlist = append(playlist, playlistName) + + // Set profile and level based on res + var profile, level string + switch outputName { + case "2160p": + profile = "high" + level = "5.1" + case "1080p": + profile = "high" + level = "4.2" + default: + profile = "main" + level = "3.1" + } + fmt.Printf("Processing %s...\n", outputName) + + // FFmpeg command arguments + ffmpegArgs := []string{ + "-y", + "-i", fileName, + "-c:v", "libx264", + "-preset", "veryfast", + "-profile:v", profile, + "-level:v", level, + "-b:v", bitrate, + "-s", res, + "-c:a", "aac", + "-b:a", "128k", + "-ac", "2", + "-g", gopSizeStr, + "-keyint_min", gopSizeStr, + "-sc_threshold", "0", + "-force_key_frames", "expr:gte(t,n_forced*4)", + "-hls_time", "4", + "-hls_list_size", "0", + "-hls_flags", "independent_segments", + "-hls_segment_filename", filepath.Join(outputDir, outputName+"_%03d.ts"), + filepath.Join(outputDir, playlistName), + } + + cmd := exec.Command("ffmpeg", ffmpegArgs...) + // Capture errors + var stderr bytes.Buffer + cmd.Stderr = &stderr + // Run command + err = cmd.Run() + if err != nil { + fmt.Printf("Error: Failed to process %s: %v\n", outputName, err) + fmt.Printf("FFmpeg error output: %s\n", stderr.String()) + return // or handle the error appropriately + } + // OKAY + // BY HERE THE HLS ENCODING IS COMPLETE + // NOW WE NEED TO UPLOAD TO R2 BUCKET + // Upload HLS files to R2 + err = uploadHLSToR2(r2Client, outputDir) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to upload HLS files: %v", err), http.StatusInternalServerError) + return + } + + } + + // HTML + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, ` @@ -71,12 +348,12 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { `, uniqueID, string(responseURL), string(responseURL)) - return - } + return + } - // Original GET form - w.Header().Set("Content-Type", "text/html") - fmt.Fprint(w, ` + // Original GET form + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, ` @@ -155,10 +432,6 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) { - `) -} -func main() { - http.HandleFunc("/", uploadHandler) - fmt.Println("Server started at localhost") - http.ListenAndServe(":3880", nil) +`) + } }