q/thirdculture
1
0
Fork 0
Code Issues 10 Pull requests Projects 1 Releases Packages Wiki Activity Actions

Compare commits

merge into: q:main
Branches Tags
q:main
q:video
...
pull from: q:video
Branches Tags
q:video
q:main

2 commits
main ... video

Author SHA1 Message Date
sam-pich
1f43b3b5c7 FFMPEG complete, upload complete, todo list at top of file 2024-11-19 16:46:34 -05:00
sam-pich
b793711537 Go file for parsing, if vid ffmpeg to HLS (inprogress) 2024-11-19 00:27:01 -05:00
4 changed files with 534 additions and 35 deletions
Show stats Expand all files Collapse all files

7
transfer/tshweb/go.mod Normal file
View file

@ -0,0 +1,7 @@
module filetransfer
go 1.23.1
require github.com/aws/aws-sdk-go v1.55.5
require github.com/jmespath/go-jmespath v0.4.0 // indirect

14
transfer/tshweb/go.sum Normal file
View file

@ -0,0 +1,14 @@
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

111
transfer/tshweb/index.html
View file

@ -1,37 +1,78 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Third Culture Upload</title>
<link rel="stylesheet" href="styles/main.css" />
</head>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Third Culture Upload</title>
<link rel="stylesheet" href="styles/main.css">
</head>
<body>
<h1><span class="accent">Third Culture</span> Upload</h1>
<form action="https://up.thirdculture.top" method="POST" enctype="multipart/form-data">
<label for="fileInput">Choose file:</label>
<input type="file" name="file" id="fileInput" required />
<br><br>
<button type="submit">Upload</button>
</form>
<p><br></p>
<p>Make sure to copy the link in the next page!</p>
<p>More info on third culture upload <a href="scripts/info.html">here</a>.</p>
<p><br></p>
<svg version="1.1" style="fill:none;width:75px;height:75px;" width="50" height="50" viewBox="0 0 50 50" id="svg3" sodipodi:docname="logo.svg"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg">
<defs id="defs3" />
<path style="fill:#000000;stroke:#000000;stroke-width:1.51515px;stroke-linecap:round;stroke-linejoin:round"
d="m 0.75757498,49.242423 v 0 V 0.75757498 H 49.242425 V 49.242423 Z" id="path1" />
<path style="fill:none;stroke:#a4d1ad;stroke-width:1.51515px;stroke-linecap:round;stroke-linejoin:round"
d="m 20.454546,38.636362 v 0 l 9.090908,-9.090909 m -9.090908,0 v 0 l 9.090908,9.090909" id="path2" />
<path style="fill:none;stroke:#aea3f0;stroke-width:4.24243px;stroke-linecap:round;stroke-linejoin:round"
d="m 6.8181812,40.151514 v 0 L 25,9.8484837 43.181819,40.151514" id="path3" />
</svg>
</body>
</html>
<body>
<h1><span class="accent">Third Culture</span> Upload</h1>
<form
action="http://localhost:3880"
method="POST"
enctype="multipart/form-data"
>
<label for="fileInput">Choose file:</label>
<input type="file" name="file" id="fileInput" required />
<br /><br />
<button type="submit">Upload</button>
</form>
<p><br /></p>
<p>Make sure to copy the link in the next page!</p>
<p>
More info on third culture upload
<a href="scripts/info.html">here</a>.
</p>
<p><br /></p>
<svg
version="1.1"
style="fill: none; width: 75px; height: 75px"
width="50"
height="50"
viewBox="0 0 50 50"
id="svg3"
sodipodi:docname="logo.svg"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg"
>
<defs id="defs3" />
<path
style="
fill: #000000;
stroke: #000000;
stroke-width: 1.51515px;
stroke-linecap: round;
stroke-linejoin: round;
"
d="m 0.75757498,49.242423 v 0 V 0.75757498 H 49.242425 V 49.242423 Z"
id="path1"
/>
<path
style="
fill: none;
stroke: #a4d1ad;
stroke-width: 1.51515px;
stroke-linecap: round;
stroke-linejoin: round;
"
d="m 20.454546,38.636362 v 0 l 9.090908,-9.090909 m -9.090908,0 v 0 l 9.090908,9.090909"
id="path2"
/>
<path
style="
fill: none;
stroke: #aea3f0;
stroke-width: 4.24243px;
stroke-linecap: round;
stroke-linejoin: round;
"
d="m 6.8181812,40.151514 v 0 L 25,9.8484837 43.181819,40.151514"
id="path3"
/>
</svg>
</body>
</html>

437
transfer/tshweb/transfer.go Normal file
View file

@ -0,0 +1,437 @@
package main
import (
"bytes"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
// NEED TO WORK ON ASYNC (once og fileuploaded to server, treat it as if its regular download, but run the ffmpeg and r2 upload in background)
// Make display page for HTML during file upload
// Make youtube esque looking page to watch the bucket files
// uploadToR2 uploads all files in the HLS output directory to R2
func uploadToR2(r2Client *s3.S3, outputDir, bucketName string) error {
// Walk through the directory and upload each file
err := filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Skip directories (not rly needed in our case)
if info.IsDir() {
return nil
}
// Open the file
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("error opening file %s: %v", path, err)
}
defer file.Close()
// Determine the key (object name) in the bucket
// This preserves the directory structure relative to the base output directory
relativePath, err := filepath.Rel(outputDir, path)
if err != nil {
return err
}
// Normalize path separators for R2
key := strings.ReplaceAll(relativePath, "\\", "/")
// Prepare the upload input, should upload to thirdculture bucket, with folder/key, and the files inside
uploadInput := &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: file,
}
// Upload the file
_, err = r2Client.PutObject(uploadInput)
if err != nil {
return fmt.Errorf("error uploading file %s to R2: %v", path, err)
}
fmt.Printf("Uploaded: %s\n", key)
return nil
})
return err
}
// NewR2Client creates a new R2 client using AWS SDK
func NewR2Client(accountId, accessKeyId, secretAccessKey string) (*s3.S3, error) {
// Cloudflare R2 endpoint follows this format
r2Endpoint := fmt.Sprintf("https://%s.r2.cloudflarestorage.com", accountId)
// Create AWS session with R2 credentials
sess, err := session.NewSession(&aws.Config{
Endpoint: aws.String(r2Endpoint),
Region: aws.String("auto"), // Cloudflare uses 'auto'
Credentials: credentials.NewStaticCredentials(accessKeyId, secretAccessKey, ""),
})
if err != nil {
return nil, fmt.Errorf("error creating AWS session: %v", err)
}
// Create S3 client (used for R2)
return s3.New(sess), nil
}
// In your existing uploadHandler function, add this after HLS conversion:
func uploadHLSToR2(r2Client *s3.S3, outputDir string) error {
// Upload to R2 bucket
err := uploadToR2(r2Client, outputDir, "thirdculture")
if err != nil {
return fmt.Errorf("error uploading HLS files to R2: %v", err)
}
// After successful upload, remove the local HLS directory
err = os.RemoveAll(outputDir)
if err != nil {
return fmt.Errorf("error deleting local HLS directory %s: %v", outputDir, err)
}
fmt.Printf("Successfully uploaded and deleted local HLS directory: %s\n", outputDir)
return nil
}
func uploadHandler(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost {
// Parse the form
err := r.ParseMultipartForm(15 * 1024 * 1024 * 1024) // limit 15 gb
if err != nil {
http.Error(w, "Unable to parse form", http.StatusBadRequest)
return
}
file, fileHeader, err := r.FormFile("file")
if err != nil {
http.Error(w, "Unable to retrieve file", http.StatusBadRequest)
return
}
defer file.Close()
// Upload the file
req, err := http.NewRequest("POST", "http://localhost:3880", file)
if err != nil {
http.Error(w, "Error creating request", http.StatusInternalServerError)
return
}
// Filetype
// Buffer needed to make it faster, so we only process one bit of the file instead of all
buffer := make([]byte, 512)
_, err = file.Read(buffer)
if err != nil && err != io.EOF {
http.Error(w, "Error reading the file", http.StatusInternalServerError)
return
}
fileType := http.DetectContentType(buffer)
file.Seek(0, 0)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
http.Error(w, "Error uploading to transfer.sh", http.StatusInternalServerError)
return
}
defer resp.Body.Close()
// Unique ID to track file for ffmpeg etc
responseURL, err := io.ReadAll(resp.Body)
if err != nil {
http.Error(w, "Error reading response", http.StatusInternalServerError)
return
}
urlParts := strings.Split(string(responseURL), "/")
uniqueID := urlParts[len(urlParts)-1]
// If its a video, proceed with encoding and seperate uploads
if strings.HasPrefix(fileType, "video/") {
// Convert file to HLS. Work on error catching and flags tomorrow
// IF VIDEO=DO THIS. IF NOT CONTINUE!
// ALSO CHANGE THE CODE. IF VIDEO, REDIRECT TO S3 BUCKET
// CURRENTLY FILE IS STANDARD UPLOAD TO SERVER
//
// Init r2 client
// API KEYS PUSHED TO GIT IS BAD, I KNOW. But this stuff isnt public and were using your git so idt anyone could crawl
// find my api keys and abuse this
// Plus this way you can access it too
r2Client, err := NewR2Client(
"58916dd30d55baca3a55ad78f0883c52", // Account
"94785bc0756ccedfe3634a7c4c7965f8", // Access
"d5666b49c672c8584bc3a393701a938ddd8412663d5ea470e2c372179a86245e", // Secret Access
)
if err != nil {
http.Error(w, "Failed to initialize R2 client", http.StatusInternalServerError)
return
}
// File vars for ffmpeg
inputDir := "../uploads"
err = os.MkdirAll(inputDir, os.ModePerm)
fileName := fileHeader.Filename
noExtensionFilename := strings.TrimSuffix(filepath.Base(fileName), filepath.Ext(fileName))
//Create output basedir
var outputBaseDir string
var outputDir string
if len(os.Args) >= 2 {
outputBaseDir = os.Args[1]
// Create dir if it doesnt exist
err := os.MkdirAll(outputBaseDir, os.ModePerm)
if err != nil {
fmt.Printf("Error creating directory: %v\n", err)
os.Exit(1)
} else {
// Default to $PWD
wd, err := os.Getwd()
if err != nil {
fmt.Printf("Error getting current directory: %v\n", err)
os.Exit(1)
}
outputBaseDir = wd
}
}
// Set specific output diir
outputDir = outputBaseDir + "/" + noExtensionFilename
// Check if the output directory exists
_, err = os.Stat(outputDir)
if err == nil {
// If directory exists, remove it
fmt.Printf("Output directory '%s' already exists. Removing it to avoid unused files.\n", outputDir)
err := os.RemoveAll(outputDir) // Removes the directory and its contents
if err != nil {
fmt.Printf("Error removing directory: %v\n", err)
return
}
}
// FFPROBE to get frame rate
cmdFFPROBE := exec.Command("ffprobe", "-v", "0", "-of", "default=noprint_wrappers=1:nokey=1",
"-select_streams", "v:0", "-show_entries", "stream=avg_frame_rate", fileName)
outputFrame, err := cmdFFPROBE.Output()
if err != nil {
fmt.Printf("Error running ffprobe: %v\n", err)
return
}
frameRateStr := strings.TrimSpace(string(outputFrame))
// Split the frame rate (which might be like "30/1" or "29.97")
parts := strings.Split(frameRateStr, "/")
var frameRateFloat float64
if len(parts) > 1 {
// Handle fraction (e.g., "30/1")
numerator, _ := strconv.ParseFloat(parts[0], 64)
denominator, _ := strconv.ParseFloat(parts[1], 64)
frameRateFloat = numerator / denominator
} else {
// Handle direct float (e.g., "29.97")
frameRateFloat, err = strconv.ParseFloat(frameRateStr, 64)
if err != nil {
fmt.Printf("Framerate parsing error: %v\n", err)
return
}
}
// Convert to int for GOP calculation
GOP_SIZE := int(frameRateFloat * 4)
gopSizeStr := strconv.Itoa(GOP_SIZE)
// resolution, bitrate, output slices/lists
resolutions := []string{"1280x720", "1920x1080", "3840x2160"}
bitrates := []string{"1200k", "2500k", "8000k"}
outputs := []string{"720p", "1080p", "2160p"}
playlist := []string{}
for i := range resolutions {
res := resolutions[i]
bitrate := bitrates[i]
outputName := outputs[i]
playlistName := outputName + ".m3u8"
playlist = append(playlist, playlistName)
// Set profile and level based on res
var profile, level string
switch outputName {
case "2160p":
profile = "high"
level = "5.1"
case "1080p":
profile = "high"
level = "4.2"
default:
profile = "main"
level = "3.1"
}
fmt.Printf("Processing %s...\n", outputName)
// FFmpeg command arguments
ffmpegArgs := []string{
"-y",
"-i", fileName,
"-c:v", "libx264",
"-preset", "veryfast",
"-profile:v", profile,
"-level:v", level,
"-b:v", bitrate,
"-s", res,
"-c:a", "aac",
"-b:a", "128k",
"-ac", "2",
"-g", gopSizeStr,
"-keyint_min", gopSizeStr,
"-sc_threshold", "0",
"-force_key_frames", "expr:gte(t,n_forced*4)",
"-hls_time", "4",
"-hls_list_size", "0",
"-hls_flags", "independent_segments",
"-hls_segment_filename", filepath.Join(outputDir, outputName+"_%03d.ts"),
filepath.Join(outputDir, playlistName),
}
cmd := exec.Command("ffmpeg", ffmpegArgs...)
// Capture errors
var stderr bytes.Buffer
cmd.Stderr = &stderr
// Run command
err = cmd.Run()
if err != nil {
fmt.Printf("Error: Failed to process %s: %v\n", outputName, err)
fmt.Printf("FFmpeg error output: %s\n", stderr.String())
return // or handle the error appropriately
}
// OKAY
// BY HERE THE HLS ENCODING IS COMPLETE
// NOW WE NEED TO UPLOAD TO R2 BUCKET
// Upload HLS files to R2
err = uploadHLSToR2(r2Client, outputDir)
if err != nil {
http.Error(w, fmt.Sprintf("Failed to upload HLS files: %v", err), http.StatusInternalServerError)
return
}
}
// HTML
w.Header().Set("Content-Type", "text/html")
fmt.Fprint(w, `
<!DOCTYPE html>
<html lang=en>
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Third Culture Upload</title>
<link rel="stylesheet" href="styles/main.css" />
</head>
<body>
<h1>Upload Success</h1>
<p><strong>Unique ID:</strong> %s</p>
<p><a href="%s" target="_blank">%s</a></p>
<a href="/">Upload Another</a>
</body>
</html>
`, uniqueID, string(responseURL), string(responseURL))
return
}
// Original GET form
w.Header().Set("Content-Type", "text/html")
fmt.Fprint(w, `
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Third Culture Upload</title>
<link rel="stylesheet" href="styles/main.css" />
</head>
<body>
<h1><span class="accent">Third Culture</span> Upload</h1>
<form
action="http://localhost:3880"
method="POST"
enctype="multipart/form-data"
>
<label for="fileInput">Choose file:</label>
<input type="file" name="file" id="fileInput" required />
<br /><br />
<button type="submit">Upload</button>
</form>
<p><br /></p>
<p>Make sure to copy the link in the next page!</p>
<p>
More info on third culture upload
<a href="scripts/info.html">here</a>.
</p>
<p><br /></p>
<svg
version="1.1"
style="fill: none; width: 75px; height: 75px"
width="50"
height="50"
viewBox="0 0 50 50"
id="svg3"
sodipodi:docname="logo.svg"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg"
>
<defs id="defs3" />
<path
style="
fill: #000000;
stroke: #000000;
stroke-width: 1.51515px;
stroke-linecap: round;
stroke-linejoin: round;
"
d="m 0.75757498,49.242423 v 0 V 0.75757498 H 49.242425 V 49.242423 Z"
id="path1"
/>
<path
style="
fill: none;
stroke: #a4d1ad;
stroke-width: 1.51515px;
stroke-linecap: round;
stroke-linejoin: round;
"
d="m 20.454546,38.636362 v 0 l 9.090908,-9.090909 m -9.090908,0 v 0 l 9.090908,9.090909"
id="path2"
/>
<path
style="
fill: none;
stroke: #aea3f0;
stroke-width: 4.24243px;
stroke-linecap: round;
stroke-linejoin: round;
"
d="m 6.8181812,40.151514 v 0 L 25,9.8484837 43.181819,40.151514"
id="path3"
/>
</svg>
</body>
</html>
`)
}
}