audio handling started

controller/tutor-shot.js

-const { STATUS_CODE, TEMP_IMAGE_DIR, TEMP_VIDEO_DIR } = require("../helper/constants")
+const { STATUS_CODE, TEMP_IMAGE_DIR, TEMP_VIDEO_DIR, TEMP_AUDIO_DIR } = require("../helper/constants")
 const { generateCanvas, mergeImages } = require("../helper/tutor-shot")
 const path = require('path')
 const sharp = require('sharp')
 const fs = require('fs')
-const { createVideoFromImages } = require("../helper/ffmpeg-helper")
-const { deleteFile } = require("../helper/utilities")
+const { createVideoFromImages, joinAudioVideo } = require("../helper/ffmpeg-helper")
+const { deleteFile, downloadFile } = require("../helper/utilities")
 const { generatePreSignedGetUrl } = require("../helper/upload")
+const mongoose = require('mongoose')
+const ObjectId = mongoose.Types.ObjectId
+const ffmpeg = require("fluent-ffmpeg");
 
 const socket = require("../socket.js")
 
 module.exports.generateVideo = async function (req, res) {
 
+    console.log('generateVideo')
+
     try {
         let tutorShotData = req.body.tutorShotData;
         let videoPath; // local path of the video file
         let thumbnailPath // local path of the thumbnail file
         let fileName; // remote name of the final video file
         let thumbnailName; // remote name of the final thumbnail
+        let updatedData;
 
         if (tutorShotData) {
+            let socketPayload = {
+                task: "Video Generation",
+                percent: 0,
+                subTask: [ { task: "Image Compiled", percent: 0 } ]
+            }
+            if ( tutorShotData?.audio?.length ){
+                socketPayload.subTask.push({ task: "Audio mixing", percent: 0 })
+            }
+            console.log('data to generate found')
 
             let imageDir = TEMP_IMAGE_DIR
             let imagePathString = path.join(imageDir, `${tutorShotData._id}_%d.png`)
@@ -31,6 +46,9 @@ module.exports.generateVideo = async function (req, res) {
             // // this is to generate each individual images and get their buffers
             let frameNumber = 1;
             let imageArr = []
+
+            socket.emit("videoGenerationProgress",  socketPayload)
+
             for (let i = 0; i < tutorShotData.segments.length; i++) {
                 if (tutorShotData.segments[i].imageUrl) tutorShotData.segments[i].signedImageUrl = await generatePreSignedGetUrl(tutorShotData.segments[i].imageUrl)
                 let CE = generateCanvas(tutorShotData.segments[i], { dimension: tutorShotData.segments[i].dimension })
@@ -60,6 +78,9 @@ module.exports.generateVideo = async function (req, res) {
                     imageArr.push(thumbnailPath)
                 }
 
+                socketPayload.subTask[0].percent = Math.round(((i+1)/tutorShotData.segments.length)*75); // 75% for generating images
+                socketPayload.percent = Math.round( tutorShotData?.audio?.length?socketPayload.subTask[0].percent/2:socketPayload.subTask[0].percent );
+                socket.emit("videoGenerationProgress",  socketPayload);
             }
 
 
@@ -72,17 +93,38 @@ module.exports.generateVideo = async function (req, res) {
                 deleteImages: true,// default is true
             }
 
-            await createVideoFromImages([], options)
+            await createVideoFromImages([], options);
+            socketPayload.subTask[0].percent = 100; // 100% if video(without audio) is generated
+            socketPayload.percent = Math.round( tutorShotData?.audio?.length?socketPayload.subTask[0].percent/2:socketPayload.subTask[0].percent );
+            socket.emit("videoGenerationProgress",  socketPayload);
+
             !(async function () {
                 deleteFile(imageArr)
             })()
 
-            socket.emit("videoGenerationProgress", {
-                totalPercent: 100, // percentage of the overall task// video generation
-                percent: 100,
-                task: "Video Generation",
-                uid: tutorShotData.uid
-            })
+            console.log('------VIDEO GENERATED------')
+            // // to save audio
+            if (tutorShotData.audio?.length) {
+                socketPayload.percent = 50
+                socketPayload.subTask[0].percent = 50
+                socket.emit("videoGenerationProgress",  socketPayload)
+                console.log('to generate audio')
+                let audioPath = path.join(TEMP_AUDIO_DIR, `ts-${tutorShotData._id}.mp3`); // path to only audio
+                console.log('audioPath', audioPath)
+                await createAudio(tutorShotData.audio, audioPath)
+                console.log('Audio created')
+                finalVideoPath = path.join(CONFIG.DEFAULT_PATH.TEMP_VIDEO, `ts-final-${tutorShotData._id}.mp4`); // path to combined video and audio
+                console.log('joining Audio and video')
+                await joinAudioVideo(videoPath, audioPath, finalVideoPath)
+                console.log('---AUDIO VIDEO JOINED----')
+            }else{
+                socketPayload.percent = 100
+                socketPayload.subTask[0].percent = 100
+                socket.emit("videoGenerationProgress",  socketPayload)
+
+            }
+
+            console.log('got updated data', updatedData)
         }
 
 
@@ -90,9 +132,92 @@ module.exports.generateVideo = async function (req, res) {
         // let remoteThumbnailPath = path.join(CONFIG.S3_PATH.TUTOR_SHOT, thumbnailName)
         // await uploadToAwsS3(videoPath, remotePath)
         // await uploadToAwsS3(thumbnailPath, remoteThumbnailPath)
-        res.status(STATUS_CODE.OK).json()
+        console.log('------SENDING RESPONSE-------')
+        res.status(STATUS_CODE.OK).json(tutorShotData)
     } catch (err) {
-        console.log('------ERROR:GENERATING VIDEO',err)
+        console.log('------ERROR:GENERATING VIDEO', err)
         res.status(STATUS_CODE.ERROR).json()
     }
 }
+
+async function createAudio(audioList, audioPath) {
+    let downloadAudioPromise = []
+    let tempFiles = []
+    for (let a = 0; a < audioList.length; a++) {
+        if (!audioList[a]?._id) {
+            audioList[a]._id = (new ObjectId()).toString()
+        }
+        audioList[a].localPath = path.join(TEMP_AUDIO_DIR, `${audioList[a]._id}.mp3`)
+        audioList[a].trimmedLocalPath = path.join(TEMP_AUDIO_DIR, `${audioList[a]._id}_trimmed.mp3`)
+        downloadAudioPromise.push(new Promise(async (response, reject) => {
+            try {
+                audioList[a].url = await generatePreSignedGetUrl(audioList[a].remotePath, audioList[a].localPath)
+                await downloadFile(audioList[a].url, audioList[a].localPath)
+                tempFiles.push(audioList[a].localPath)
+                if (audioList[a].trim) {
+                    ffmpeg()
+                        .input(audioList[a].localPath)
+                        .seekInput(0)
+                        .duration(audioList[a].trim)
+                        .audioCodec('copy') // Copy audio codec without re-encoding
+                        .output(audioList[a].trimmedLocalPath)
+                        .on('end', () => {
+                            console.log('Trimming finished!');
+                            response(audioList[a].localPath)
+                        })
+                        .on('error', (err) => {
+                            console.error('Error:', err);
+                            reject(err)
+                        })
+                        .run();
+
+                } else {
+                    response(audioList[a].localPath)
+                }
+            } catch (err) {
+                reject(err)
+            }
+        }))
+    }
+
+    await Promise.all(downloadAudioPromise)
+    console.log('AUDIO files Created')
+
+
+    await new Promise((res, rej) => {
+
+        let audioObj = ffmpeg()
+        audioObj.on('end', () => {
+            deleteFile(tempFiles)
+            tempFiles.push(audioPath)
+            res()
+        }).on('error', (err) => {
+            deleteFile(tempFiles)
+            console.error('----------Error:\n', err);
+            rej(err)
+        })
+
+        for (let a = 0; a < audioList.length; a++) {
+            let prevEndTime = 0
+            if (a != 0) prevEndTime = audioList[a - 1].trim ? audioList[a - 1].trim : (audioList[a - 1].endTime)
+            if (prevEndTime) {
+                const silenceAudio = path.join(TEMP_AUDIO_DIR, `temp-${audioList[a]._id}.mp3`);
+                let waitTime = audioList[a].startTime - prevEndTime; // in seconds
+                if (audioList[a].trim) {
+                    waitTime = audioList[a].trim
+                }
+                let command = `ffmpeg -f lavfi -i anullsrc=r=44100 -t ${waitTime} -c:a libmp3lame -q:a 0 ${silenceAudio} -y`
+                execSync(command)
+                console.log(a, 'Mute audio created')
+                audioObj.input(silenceAudio).inputOptions('-ss', '0');
+                tempFiles.push(silenceAudio)
+            }
+            audioObj.input(audioList[a].trim ? audioList[a].trimmedLocalPath : audioList[a].localPath).inputOptions('-ss', '0');
+        }
+
+        audioObj.mergeToFile(audioPath, path.dirname(audioPath))
+        console.log('audio merged')
+    })
+
+    return
+}
\ No newline at end of file

helper/utilities.js

 const fs = require('fs')
-
+const axios = require("axios")
 
 module.exports.createDirectory = function (directory) {
     fs.mkdir(directory, { recursive: true }, (err) => {
@@ -27,3 +27,26 @@ module.exports.deleteFile = function (item) {
     }
     return
 }
+
+
+module.exports.downloadFile = async function (url, localPath) {
+    try {
+        const response = await axios({
+            method: 'GET',
+            url: url,
+            responseType: 'stream', // Important to handle binary data
+        });
+
+        // Create a writable stream and pipe the response data to it
+        const writer = fs.createWriteStream(localPath);
+        response.data.pipe(writer);
+
+        return new Promise((resolve, reject) => {
+            writer.on('finish', resolve);
+            writer.on('error', reject);
+        });
+    } catch (error) {
+        console.error('Error:', error.message);
+        throw error;
+    }
+}
\ No newline at end of file

model/index.js

-const fs = require('fs');
-const path = require('path');
-const basename = path.basename(__filename);
-const mongoose = require('mongoose');
-
-const models = {};
-fs
-    .readdirSync(__dirname)
-    .filter(file => {
-        return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
-    })
-    .forEach(async file => {
-        const schema = require(path.join(__dirname, file));
-        const model = mongoose.model(schema.collectionName, schema.collectionSchema)
-        models[schema.collectionName] = model
-    });
-
-module.exports = models;
\ No newline at end of file

model/tutor-shot.js

-const mongoose = require("mongoose");
-
-let segmentSchema = {
-
-    "tutorShotOid": mongoose.Schema.Types.ObjectId,
-    "title": String,
-    "action": { type: String, default: "SSCaptured" },
-    "imageUrl": String,
-    "thumbnailUrl": String,
-    "imageNumber": Number,
-    "url": String,
-    "origin": String,
-    "object": [
-        {
-            "dimension": {
-                "width": { type: Number, default: 400 },
-                "height": { type: Number, default: 197 }
-            },
-            "position": {
-                "top": { type: Number, default: 90 },
-                "left": { type: Number, default: 768 }
-            },
-            "font": {
-                "size": { type: Number, default: 37 },
-                "family": { type: String, default: "Arial" },
-                "color": { type: String, default: "#ff1245" }
-            },
-            "type": { type: String, default: "block" },
-            "text": { type: String, default: "" },
-            "transform": {
-                "xFlip": { type: Boolean, default: false },
-                "yFlip": { type: Boolean, default: false },
-                "angle": { type: Number, default: 0 }
-            },
-            "format": {
-                "backgroundColor": { type: String, default: "#0000007f" },
-                "borderRadius": { type: Number, default: 7 },
-                "borderWidth": { type: Number, default: 10 },
-                "borderColor": { type: String, default: "#af7f1c" }
-            },
-            "image": {
-                "src": String
-            },
-            "id": String
-        }
-    ],
-    "duration": { type: Number, default: 2 }, // in seconds
-    "manualDuration": { type: Boolean, default: false },
-    "dimension": Object,
-    "deleteStatus": Boolean,
-}
-let audioSchema = {
-    uploadId: mongoose.Types.ObjectId,
-    originalName: String,
-    fileName: String,
-    remotePath: String,
-    segmentId: mongoose.Types.ObjectId, // ref to the segment object in the segment field
-    objectId: mongoose.Types.ObjectId, // ref to the object inside each segment inside segment field
-    trackNumber: { type: Number, default: 1 },
-    startTime: { type: Number, default: 0 },
-    endTime: { type: Number, default: 0 },
-    skip: { type: Boolean, default: false }, // to use the audio or not while generating video
-    trim: { type: Number } // sometime we would need only a part of a video. This stores the length of the video required 
-}
-let collectionSchema = new mongoose.Schema(
-    {
-        uid: { type: String }, // id of the user
-        // fullName: { type: String }, // name of the user
-        // email: { type: String }, // email of the user
-        segments: [segmentSchema],
-        audio: [audioSchema],
-
-        title: { type: String }, // this is constant
-        deleteStatus: { type: Boolean, default: false },
-        videoUrl: String,
-        thumbnailUrl: String,
-    },
-    { timestamps: true }
-)
-module.exports = { collectionSchema, collectionName: "tutorShot" }