FF-261 - added audio to video node

VideoNodes/FfmpegBuilderNodes/Video/FfmpegBuilderVideoEncode.cs

@@ -85,10 +85,13 @@ public class FfmpegBuilderVideoEncode:FfmpegBuilderNode
     public override int Execute(NodeParameters args)
     {
         var stream = Model.VideoStreams.Where(x => x.Deleted == false).First();
+
+        stream.EncodingParameters.Clear();        
+
         if (Codec == CODEC_H264 || Codec == CODEC_H264_10BIT)
-            H264(stream, Codec == CODEC_H264_10BIT);
+            stream.EncodingParameters.AddRange(H264(args, Codec == CODEC_H264_10BIT, Quality, HardwareEncoding));
         else if (Codec == CODEC_H265 || Codec == CODEC_H265_10BIT)
-            H265(stream, Codec == CODEC_H265_10BIT);
+            stream.EncodingParameters.AddRange(H265(args, Codec == CODEC_H265_10BIT, Quality, HardwareEncoding));
         else
         {
             args.Logger?.ILog("Unknown codec: " + Codec);
@@ -99,88 +102,106 @@ public class FfmpegBuilderVideoEncode:FfmpegBuilderNode
         return 1;
     }
 
-    private void H264(FfmpegVideoStream stream, bool tenBit)
+    internal static IEnumerable<string> GetEncodingParameters(NodeParameters args, string codec, int quality, bool useHardwareEncoder)
     {
-        if (HardwareEncoding == false)
-            H26x_CPU(stream, false);
-        else if (CanUseHardwareEncoding.CanProcess_Nvidia_H264(Args))
-            H26x_Nvidia(stream, false);
-        else if (CanUseHardwareEncoding.CanProcess_Qsv_H264(Args))
-            H26x_Qsv(stream, false);
-        else if (CanUseHardwareEncoding.CanProcess_Amd_H264(Args))
-            H26x_Amd(stream, false);
-        else if (CanUseHardwareEncoding.CanProcess_Vaapi_H264(Args))
-            H26x_Vaapi(stream, false);
-        else 
-            H26x_CPU(stream, false);
+        if (codec == CODEC_H264 || codec == CODEC_H264_10BIT)
+            return H264(args, codec == CODEC_H264_10BIT, quality, useHardwareEncoder).Select(x => x.Replace("{index}", "0"));
+        else if (codec == CODEC_H265 || codec == CODEC_H265_10BIT)
+            return H265(args, codec == CODEC_H265_10BIT, quality, useHardwareEncoder).Select(x => x.Replace("{index}", "0"));
+        throw new Exception("Unsupported codec: " + codec);
+
+    }
+
+    private static IEnumerable<string> H264(NodeParameters args, bool tenBit, int quality, bool useHardwareEncoding)
+    {
+        List<string> parameters = new List<string>();
+        string[] bit10Filters = null;
+        string[] non10BitFilters = null;
+        if (useHardwareEncoding == false)
+            parameters.AddRange(H26x_CPU(false, quality, out bit10Filters));
+        else if (CanUseHardwareEncoding.CanProcess_Nvidia_H264(args))
+            parameters.AddRange(H26x_Nvidia(false, quality, out non10BitFilters));
+        else if (CanUseHardwareEncoding.CanProcess_Qsv_H264(args))
+            parameters.AddRange(H26x_Qsv(false, quality));
+        else if (CanUseHardwareEncoding.CanProcess_Amd_H264(args))
+            parameters.AddRange(H26x_Amd(false, quality));
+        else if (CanUseHardwareEncoding.CanProcess_Vaapi_H264(args))
+            parameters.AddRange(H26x_Vaapi(false, quality));
+        else
+            parameters.AddRange(H26x_CPU(false, quality, out bit10Filters));
 
         if (tenBit)
-            stream.EncodingParameters.AddRange(bit10Filters);
+            parameters.AddRange(bit10Filters ?? new string[] { "-pix_fmt:v:{index}", "p010le", "-profile:v:{index}", "main10" });
+        return parameters;
     }
     
-    private void H265(FfmpegVideoStream stream, bool tenBit)
+    private static IEnumerable<string> H265(NodeParameters args, bool tenBit, int quality, bool useHardwareEncoding)
     {
         // hevc_qsv -load_plugin hevc_hw -pix_fmt p010le -profile:v main10 -global_quality 21 -g 24 -look_ahead 1 -look_ahead_depth 60
-        if (HardwareEncoding == false)
-            H26x_CPU(stream, true);
-        else if (CanUseHardwareEncoding.CanProcess_Nvidia_Hevc(Args))
-            H26x_Nvidia(stream, true);
-        else if (CanUseHardwareEncoding.CanProcess_Qsv_Hevc(Args))
-            H26x_Qsv(stream, true);
-        else if (CanUseHardwareEncoding.CanProcess_Amd_Hevc(Args))
-            H26x_Amd(stream, true);
-        else if (CanUseHardwareEncoding.CanProcess_Vaapi_Hevc(Args))
-            H26x_Vaapi(stream, true);
-        else 
-            H26x_CPU(stream, true);
+        List<string> parameters = new List<string>();
+        string[] bit10Filters = null;
+        string[] non10BitFilters = null;
+        if (useHardwareEncoding == false)
+            parameters.AddRange(H26x_CPU(true, quality, out bit10Filters));
+        else if (CanUseHardwareEncoding.CanProcess_Nvidia_Hevc(args))
+            parameters.AddRange(H26x_Nvidia(true, quality, out non10BitFilters));
+        else if (CanUseHardwareEncoding.CanProcess_Qsv_Hevc(args))
+            parameters.AddRange(H26x_Qsv(true, quality));
+        else if (CanUseHardwareEncoding.CanProcess_Amd_Hevc(args))
+            parameters.AddRange(H26x_Amd(true, quality));
+        else if (CanUseHardwareEncoding.CanProcess_Vaapi_Hevc(args))
+            parameters.AddRange(H26x_Vaapi(true, quality));
+        else
+            parameters.AddRange(H26x_CPU(true, quality, out bit10Filters));
 
         if (tenBit)
-            stream.EncodingParameters.AddRange(bit10Filters);
+            parameters.AddRange(bit10Filters ?? new string[] { "-pix_fmt:v:{index}", "p010le", "-profile:v:{index}", "main10" });
         else if(non10BitFilters?.Any() == true)
-            stream.EncodingParameters.AddRange(non10BitFilters);        
+            parameters.AddRange(non10BitFilters);
+        return parameters;
     }
 
 
-    private void H26x_CPU(FfmpegVideoStream stream, bool h265)
+    private static IEnumerable<string> H26x_CPU(bool h265, int quality, out string[] bit10Filters)
     {
-        stream.EncodingParameters.Clear();
-        stream.EncodingParameters.AddRange(new []
-        {
-            h265 ? "libx265" : "libx264",
-            "-preset", "slow",
-            "-crf", Quality.ToString()
-        });
         bit10Filters = new[]
         {
             "-pix_fmt:v:{index}", "yuv420p10le", "-profile:v:{index}", "main10"
         };
+        return new []
+        {
+            h265 ? "libx265" : "libx264",
+            "-preset", "slow",
+            "-crf", quality.ToString()
+        };
     }
 
-    private void H26x_Nvidia(FfmpegVideoStream stream, bool h265)
+    private static IEnumerable<string> H26x_Nvidia(bool h265, int quality, out string[] non10BitFilters)
     {
-        stream.EncodingParameters.Clear();
-        stream.EncodingParameters.AddRange(new []
+        if (h265 == false)
+            non10BitFilters = new[] { "-pix_fmt:v:{index}", "yuv420p" };
+        else
+            non10BitFilters = null;
+
+        return new []
         {
             h265 ? "hevc_nvenc" : "h264_nvenc",
             "-rc", "constqp",
-            "-qp", Quality.ToString(),
+            "-qp", quality.ToString(),
             //"-b:v", "0K", // this would do a two-pass... slower
             "-preset", "p6",
             // https://www.reddit.com/r/ffmpeg/comments/gg5szi/what_is_spatial_aq_and_temporal_aq_with_nvenc/
             "-spatial-aq", "1"
-        });
-        if (h265 == false) {
-            non10BitFilters = new[] { "-pix_fmt:v:{index}", "yuv420p" };
-        }
+        };
     }
 
-    private void H26x_Qsv(FfmpegVideoStream stream, bool h265)
+    private static IEnumerable<string> H26x_Qsv(bool h265, int quality)
     {
         //hevc_qsv -load_plugin hevc_hw -pix_fmt p010le -profile:v main10 -global_quality 21 -g 24 -look_ahead 1 -look_ahead_depth 60
-        stream.EncodingParameters.Clear();
+        var parameters = new List<string>();
         if (h265) 
         {
-            stream.EncodingParameters.AddRange(new[]
+            parameters.AddRange(new[]
             {
                 "hevc_qsv",
                 "-load_plugin", "hevc_hw"
@@ -188,43 +209,42 @@ public class FfmpegBuilderVideoEncode:FfmpegBuilderNode
         }
         else
         {
-            stream.EncodingParameters.AddRange(new[]
+            parameters.AddRange(new[]
             {
                 "h264_qsv"
             });
 
         }
-        stream.EncodingParameters.AddRange(new[]
+        parameters.AddRange(new[]
         {
-            "-global_quality", Quality.ToString(),
+            "-global_quality", quality.ToString(),
             "-preset", "slower",
         });
+        return parameters;
     }
 
-    private void H26x_Amd(FfmpegVideoStream stream, bool h265)
+    private static IEnumerable<string> H26x_Amd(bool h265, int quality)
     {
-        stream.EncodingParameters.Clear();
-        stream.EncodingParameters.AddRange(new[]
+        return new[]
         {
             h265 ? "hevc_amf" : "h264_amf",
-            "-qp", Quality.ToString(),
+            "-qp", quality.ToString(),
             //"-b:v", "0K", // this would do a two-pass... slower
             "-preset", "slower",
             // https://www.reddit.com/r/ffmpeg/comments/gg5szi/what_is_spatial_aq_and_temporal_aq_with_nvenc/
             "-spatial-aq", "1"
-        });
+        };
     }
-    private void H26x_Vaapi(FfmpegVideoStream stream, bool h265)
+    private static IEnumerable<string> H26x_Vaapi(bool h265, int quality)
     {
-        stream.EncodingParameters.Clear();
-        stream.EncodingParameters.AddRange(new[]
+        return new[]
         {
             h265 ? "hevc_vaapi" : "h264_vaapi",
-            "-qp", Quality.ToString(),
+            "-qp", quality.ToString(),
             //"-b:v", "0K", // this would do a two-pass... slower
             "-preset", "slower",
             // https://www.reddit.com/r/ffmpeg/comments/gg5szi/what_is_spatial_aq_and_temporal_aq_with_nvenc/
             "-spatial-aq", "1"
-        });
+        };
     }
 }

VideoNodes/Tests/AudioToVideoTests.cs

@@ -0,0 +1,57 @@
+#if(DEBUG)
+
+namespace VideoNodes.Tests;
+
+using FileFlows.VideoNodes;
+using FileFlows.VideoNodes.FfmpegBuilderNodes;
+using FileFlows.VideoNodes.VideoNodes;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+[TestClass]
+public class AudioToVideoTests : TestBase
+{
+    [TestMethod]
+    public void AudioToVideo_Waves_h265()
+        => TestStyle(FfmpegBuilderVideoEncode.CODEC_H264, AudioToVideo.VisualizationStyle.Waves);
+
+    [TestMethod]
+    public void AudioToVideo_AudioVectorScope_H265()
+        => TestStyle(FfmpegBuilderVideoEncode.CODEC_H265, AudioToVideo.VisualizationStyle.AudioVectorScope);
+
+
+    [TestMethod]
+    public void AudioToVideo_Spectrum_H265_10Bit()
+        => TestStyle(FfmpegBuilderVideoEncode.CODEC_H265_10BIT, AudioToVideo.VisualizationStyle.Spectrum);
+
+
+
+    private void TestStyle(string codec, AudioToVideo.VisualizationStyle style)
+    {
+        var logger = new TestLogger();
+        string file = @"D:\music\unprocessed\01-billy_joel-movin_out.mp3";
+        var vi = new VideoInfoHelper(FfmpegPath, logger);
+        var vii = vi.Read(file);
+
+        var args = new NodeParameters(file, logger, false, string.Empty);
+        args.GetToolPathActual = (string tool) => FfmpegPath;
+        args.TempPath = TempPath;
+
+
+        AudioToVideo node = new();
+        node.Container = "mkv";
+        node.Resolution = "1280x720";
+        node.Codec = codec;
+        node.HardwareEncoding = true;
+        node.Visualization = style;
+        if (node.Visualization == AudioToVideo.VisualizationStyle.Waves)
+            node.Color = "#007bff";
+        node.PreExecute(args);
+        int output = node.Execute(args);
+
+        var log = logger.ToString();
+        Assert.AreEqual(1, output);
+    }
+}
+
+
+#endif

VideoNodes/VideoNodes.en.json

@@ -7,6 +7,20 @@
   "7": { "1": "7.1" },
   "Flow": {
     "Parts": {
+      "AudioToVideo": {
+        "Description": "Converts an audio file into a video file and generates a video based on the audio",
+        "Fields": {
+          "Visualisation": "Visualisation",
+          "Visualisation-Help": "The visualation to use in the generated video.  See Help for examples",
+          "Container": "Container",
+          "Resolution": "Resolution",
+          "Codec": "Codec",
+          "HardwareEncoding": "Hardware Encode",
+          "HardwareEncoding-Help": "When checked, will test to see if hardware encoders are found on the Processing Node, and if found will use hardware encoding, otherwise will fallback to CPU encoding.",
+          "Color": "Color",
+          "Color-Help": "Optional #RRGGBB color code to use for the sound waves, must be in the format #RRGGBB, for example #FF0090"
+        }
+      },
       "CanUseHardwareEncoding": {
         "Description": "Checks if the specified hardware encoder is currently available to the Flow.",
         "Fields": {

VideoNodes/VideoNodes/AudioToVideo.cs

@@ -0,0 +1,161 @@
+using FileFlows.VideoNodes.FfmpegBuilderNodes;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FileFlows.VideoNodes.VideoNodes;
+
+/// <summary>
+/// Node that converts a audio file into a video file and generates a video based on the audio
+/// </summary>
+public class AudioToVideo : EncodingNode
+{
+    public override int Outputs => 1;
+    public override int Inputs => 1;
+    public override string HelpUrl => "https://docs.fileflows.com/plugins/video-nodes/audio-to-video";
+
+    public override string Icon => "fas fa-headphones";
+
+    public enum VisualizationStyle
+    {
+        Waves = 1,
+        AudioVectorScope = 2,
+        Spectrum = 3
+    }
+
+    [DefaultValue(VisualizationStyle.Waves)]
+    [Select(nameof(VisualizationOptions), 1)]
+    public VisualizationStyle Visualization { get; set; }
+
+    private static List<ListOption> _VisualisationOptions;
+    public static List<ListOption> VisualizationOptions
+    {
+        get
+        {
+            if (_VisualisationOptions == null)
+            {
+                _VisualisationOptions = new List<ListOption>
+                    {
+                        new ListOption { Label = "Waves", Value = VisualizationStyle.Waves },
+                        new ListOption { Label = "Audio Vector Scope", Value = VisualizationStyle.AudioVectorScope },
+                        new ListOption { Label = "Spectrum", Value = VisualizationStyle.Spectrum },
+                    };
+            }
+            return _VisualisationOptions;
+        }
+    }
+
+
+    [DefaultValue("mkv")]
+    [Select(nameof(ContainerOptions), 2)]
+    public string Container { get; set; } = string.Empty;
+
+    private static List<ListOption> _ContainerOptions;
+    public static List<ListOption> ContainerOptions
+    {
+        get
+        {
+            if (_ContainerOptions == null)
+            {
+                _ContainerOptions = new List<ListOption>
+                    {
+                        new ListOption { Label = "MKV", Value = "mkv"},
+                        new ListOption { Label = "MP4", Value = "mp4"}
+                    };
+            }
+            return _ContainerOptions;
+        }
+    }
+
+    [DefaultValue("1280x720")]
+    [Select(nameof(ResolutionOptions), 3)]
+    public string Resolution { get; set; }
+
+    private static List<ListOption> _ResolutionOptions;
+    public static List<ListOption> ResolutionOptions
+    {
+        get
+        {
+            if (_ResolutionOptions == null)
+            {
+                _ResolutionOptions = new List<ListOption>
+                    {
+                        new ListOption { Label = "480p", Value = "640x480"},
+                        new ListOption { Label = "720p", Value = "1280x720"},
+                        new ListOption { Label = "1080p", Value = "1920x1080"},
+                        new ListOption { Label = "4K", Value = "3840x2160"}
+                    };
+            }
+            return _ResolutionOptions;
+        }
+    }
+
+    [DefaultValue(FfmpegBuilderVideoEncode.CODEC_H264)]
+    [Select(nameof(CodecOptions), 4)]
+    public string Codec { get; set; }
+
+    private static List<ListOption> _CodecOptions;
+    /// <summary>
+    /// Gets or sets the codec options
+    /// </summary>
+    public static List<ListOption> CodecOptions
+    {
+        get
+        {
+            if (_CodecOptions == null)
+            {
+                _CodecOptions = new List<ListOption>
+                {
+                    new () { Label = "H.264", Value = FfmpegBuilderVideoEncode.CODEC_H264 },
+                    //new () { Label = "H.264 (10-Bit)", Value = FfmpegBuilderVideoEncode.CODEC_H264_10BIT },
+                    new () { Label = "H.265", Value = FfmpegBuilderVideoEncode.CODEC_H265 },
+                    new () { Label = "H.265 (10-Bit)", Value = FfmpegBuilderVideoEncode.CODEC_H265_10BIT },
+                };
+            }
+            return _CodecOptions;
+        }
+    }
+
+    [Boolean(5)]
+    [DefaultValue(true)]
+    public bool HardwareEncoding { get; set; }
+
+    [TextVariable(6)]
+    [DefaultValue("#ff0090")]
+    [ConditionEquals(nameof(Visualization), VisualizationStyle.Waves)]
+    public string Color { get; set; }
+
+    public override int Execute(NodeParameters args)
+    {
+        List<string> ffArgs = new List<string>();
+        var encodingParameters = FfmpegBuilderVideoEncode.GetEncodingParameters(args, this.Codec, 28, HardwareEncoding);
+
+        switch (Visualization)
+        {
+            case VisualizationStyle.Waves:
+                var color = this.Color;
+                if (Regex.IsMatch(color ?? String.Empty, "#[0-9a-fA-F]{6}") == false)
+                    color = "#ff0090"; // use default colour
+                ffArgs.AddRange(new[] { "-filter_complex", $"[0:a]showwaves=s={Resolution}:mode=line:s=hd1080:colors={color}[v]" });
+                break;
+            case VisualizationStyle.AudioVectorScope:
+                ffArgs.AddRange(new[] { "-filter_complex", $"[0:a]avectorscope=s={Resolution}[v]" });
+                break;
+            case VisualizationStyle.Spectrum:
+                ffArgs.AddRange(new[] { "-filter_complex", $"[0:a]showspectrum=s={Resolution}:mode=separate:color=intensity:slide=1:scale=cbrt[v]" });
+                break;
+        }
+        ffArgs.AddRange(new[] { "-map", "[v]" });
+        ffArgs.Add("-vcodec");
+        ffArgs.AddRange(encodingParameters);
+        ffArgs.AddRange(new[] { "-map", "0:a" });
+        if (base.Encode(args, base.FFMPEG, ffArgs, this.Container) == false)
+        {
+            args.Logger?.ELog("Failed to encode");
+            return -1;
+        }
+        return 1;
+    }
+}

VideoNodes/VideoNodes/VideoNode.cs

@@ -118,6 +118,8 @@ namespace FileFlows.VideoNodes
         protected VideoInfo GetVideoInfo(NodeParameters args, bool refreshIfFileChanged = true)
         {
             var vi = GetVideoInfoActual(args);
+            if(vi == null) return null;
+
             if (refreshIfFileChanged == false || vi.FileName == args.FileName)
                 return vi;