FF-1187 - ffmpeg executor now reports additional info

This commit is contained in:
John Andrews
2024-01-24 13:17:54 +13:00
parent a54c10d6b7
commit 38a07c66ce
6 changed files with 567 additions and 29 deletions

View File

@@ -11,7 +11,7 @@ namespace ChecksumNodes.Tests
[TestMethod]
public void Checksum_MD5_Large()
{
var args = new NodeParameters(@"D:\videos\Injustice.mkv", new TestLogger(), false, "");
var args = new NodeParameters(@"D:\videos\Injustice.mkv", new TestLogger(), false, "", null);
var output = new MD5().Execute(args);
Assert.IsTrue(args.Variables.ContainsKey("MD5"));
Assert.IsFalse(string.IsNullOrWhiteSpace(args.Variables["MD5"] as string));
@@ -20,7 +20,7 @@ namespace ChecksumNodes.Tests
[TestMethod]
public void Checksum_SHA1_Large()
{
var args = new NodeParameters(@"D:\videos\Injustice.mkv", new TestLogger(), false, "");
var args = new NodeParameters(@"D:\videos\Injustice.mkv", new TestLogger(), false, "", null);
var output = new SHA1().Execute(args);
Assert.IsTrue(args.Variables.ContainsKey("SHA1"));
Assert.IsFalse(string.IsNullOrWhiteSpace(args.Variables["SHA1"] as string));
@@ -29,7 +29,7 @@ namespace ChecksumNodes.Tests
[TestMethod]
public void Checksum_SHA256_Large()
{
var args = new NodeParameters(@"D:\videos\Injustice.mkv", new TestLogger(), false, "");
var args = new NodeParameters(@"D:\videos\Injustice.mkv", new TestLogger(), false, "", null);
var output = new SHA256().Execute(args);
Assert.IsTrue(args.Variables.ContainsKey("SHA256"));
Assert.IsFalse(string.IsNullOrWhiteSpace(args.Variables["SHA256"] as string));
@@ -38,7 +38,7 @@ namespace ChecksumNodes.Tests
[TestMethod]
public void Checksum_SHA512_Large()
{
var args = new NodeParameters(@"D:\videos\Injustice.mkv", new TestLogger(), false, "");
var args = new NodeParameters(@"D:\videos\Injustice.mkv", new TestLogger(), false, "", null);
var output = new SHA512().Execute(args);
Assert.IsTrue(args.Variables.ContainsKey("SHA512"));
Assert.IsFalse(string.IsNullOrWhiteSpace(args.Variables["SHA512"] as string));

View File

@@ -14,9 +14,14 @@ namespace FileFlows.VideoNodes
TaskCompletionSource<bool> outputCloseEvent, errorCloseEvent;
private Regex rgxTime = new Regex(@"(?<=(time=))([\d]+:?)+\.[\d]+");
private Regex rgxFps = new Regex(@"(?<=(fps=[\s]?))([\d]+)");
private Regex rgxSpeed = new Regex(@"(?<=(speed=[\s]?))([\d]+(\.[\d]+)?)");
private Regex rgxBitrate = new Regex(@"(?<=(bitrate=[\s]?))([\d]+(\.[\d]+)?)kbits");
public delegate void TimeEvent(TimeSpan time);
public event TimeEvent AtTime;
public delegate void StatChange(string name, object value);
public event StatChange OnStatChange;
private Process process;
@@ -90,6 +95,37 @@ namespace FileFlows.VideoNodes
{
var result = new ProcessResult();
var hwDecoderIndex = arguments.FindIndex(x => x == "-hwaccel");
if (hwDecoderIndex >= 0 && hwDecoderIndex < arguments.Count - 2)
{
var decoder = arguments[hwDecoderIndex + 1].ToLowerInvariant();
foreach(var dec in new []
{
("qsv", "QSV"), ("cuda", "NVIDIA"), ("amf", "AMD"), ("vulkan", "Vulkan"), ("vaapi", "VAAPI"), ("dxva2", "dxva2"),
("d3d11va", "d3d11va"), ("opencl", "opencl")
})
{
if (decoder == dec.Item1)
{
OnStatChange?.Invoke("Decoder", dec.Item2);
break;
}
}
}
if(arguments.Any(x => x.ToLowerInvariant().Contains("hevc_qsv") || x.ToLowerInvariant().Contains("h264_qsv") || x.ToLowerInvariant().Contains("av1_qsv")))
OnStatChange?.Invoke("Encoder", "QSV");
else if(arguments.Any(x => x.ToLowerInvariant().Contains("_nvenc")))
OnStatChange?.Invoke("Encoder", "NVIDIA");
else if(arguments.Any(x => x.ToLowerInvariant().Contains("_amf")))
OnStatChange?.Invoke("Encoder", "AMD");
else if(arguments.Any(x => x.ToLowerInvariant().Contains("_vaapi")))
OnStatChange?.Invoke("Encoder", "VAAPI");
else if(arguments.Any(x => x.ToLowerInvariant().Contains("_videotoolbox")))
OnStatChange?.Invoke("Encoder", "VideoToolbox");
else if(arguments.Any(x => x.ToLowerInvariant().Contains("libx") || x.ToLowerInvariant().Contains("libvpx")))
OnStatChange?.Invoke("Encoder", "CPU");
using (var process = new Process())
{
this.process = process;
@@ -174,6 +210,7 @@ namespace FileFlows.VideoNodes
return result;
}
public void OnOutputDataReceived(object sender, DataReceivedEventArgs e)
{
// The output stream has been closed i.e. the process has terminated
@@ -183,18 +220,7 @@ namespace FileFlows.VideoNodes
}
else
{
if (e.Data.Contains("Skipping NAL unit"))
return; // just slighlty ignore these
if (rgxTime.IsMatch(e.Data))
{
var timeString = rgxTime.Match(e.Data).Value;
var ts = TimeSpan.Parse(timeString);
Logger.DLog("TimeSpan Detected: " + ts);
if (AtTime != null)
AtTime.Invoke(ts);
}
Logger.ILog(e.Data);
outputBuilder.AppendLine(e.Data);
CheckOutputLine(e.Data);
}
}
@@ -210,22 +236,45 @@ namespace FileFlows.VideoNodes
Logger.ELog(e.Data);
errorBuilder.AppendLine(e.Data);
}
else if (e.Data.Contains("Skipping NAL unit"))
else
{
return; // just slighlty ignore these
CheckOutputLine(e.Data);
}
}
private void CheckOutputLine(string line)
{
if (line.Contains("Skipping NAL unit"))
return; // just slightly ignore these
if (rgxTime.IsMatch(line))
{
var timeString = rgxTime.Match(line).Value;
var ts = TimeSpan.Parse(timeString);
Logger.DLog("TimeSpan Detected: " + ts);
AtTime?.Invoke(ts);
}
if (line.Contains("Exit Code"))
{
OnStatChange?.Invoke("Speed", null);
OnStatChange?.Invoke("Bitrate", null);
OnStatChange?.Invoke("FPS", null);
}
else
{
if (rgxTime.IsMatch(e.Data))
{
var timeString = rgxTime.Match(e.Data).Value;
var ts = TimeSpan.Parse(timeString);
if (AtTime != null)
AtTime.Invoke(ts);
}
Logger.ILog(e.Data);
outputBuilder.AppendLine(e.Data);
if (rgxSpeed.TryMatch(line, out Match matchSpeed))
OnStatChange?.Invoke("Speed", matchSpeed.Value);
if (rgxBitrate.TryMatch(line, out Match matchBitrate))
OnStatChange?.Invoke("Bitrate", matchBitrate.Value);
if (rgxBitrate.TryMatch(line, out Match matchFps) && int.TryParse(matchFps.Value, out int fps))
OnStatChange?.Invoke("FPS", fps);
}
Logger.ILog(line);
outputBuilder.AppendLine(line);
}

View File

@@ -19,7 +19,7 @@ public class FfmpegBuilder_VideoEncode_VideoEncodeTests: TestBase
string ffmpeg = FfmpegPath;
var vi = new VideoInfoHelper(ffmpeg, logger);
var vii = vi.Read(file);
var args = new NodeParameters(file, logger, false, string.Empty, null);
var args = new NodeParameters(file, logger, false, string.Empty, new LocalFileService());
args.GetToolPathActual = (string tool) => ffmpeg;
args.TempPath = TempPath;
args.Parameters.Add("VideoInfo", vii);
@@ -54,7 +54,7 @@ public class FfmpegBuilder_VideoEncode_VideoEncodeTests: TestBase
h265 ? FfmpegBuilderVideoEncode.CODEC_H265 :
FfmpegBuilderVideoEncode.CODEC_H264;
var result = Encode(codec, quality, hardware, TestFile_120_mbps_4k_uhd_hevc_10bit,
var result = Encode(codec, quality, hardware, TestFile_Sitcom,
$"{(hardware ? "nvidia" : "cpu")}_h26{(h265 ? "5" : "4")}{(bit10 ? "_10bit" : "")}_{quality}.mkv");
}

View File

@@ -0,0 +1,483 @@
#if(DEBUG)
using FileFlows.Plugin;
using FileFlows.Plugin.Models;
using FileFlows.Plugin.Services;
using System.IO;
using FileHelper = FileFlows.Plugin.Helpers.FileHelper;
namespace VideoNodes.Tests;
public class LocalFileService : IFileService
{
/// <summary>
/// Gets or sets the path separator for the file system
/// </summary>
public char PathSeparator { get; init; } = Path.DirectorySeparatorChar;
/// <summary>
/// Gets or sets the allowed paths the file service can access
/// </summary>
public string[] AllowedPaths { get; init; }
/// <summary>
/// Gets or sets a function for replacing variables in a string.
/// </summary>
/// <remarks>
/// The function takes a string input, a boolean indicating whether to strip missing variables,
/// and a boolean indicating whether to clean special characters.
/// </remarks>
public ReplaceVariablesDelegate ReplaceVariables { get; set; }
/// <summary>
/// Gets or sets the permissions to use for files
/// </summary>
public int? Permissions { get; set; }
/// <summary>
/// Gets or sets the owner:group to use for files
/// </summary>
public string OwnerGroup { get; set; }
/// <summary>
/// Gets or sets the logger used for logging
/// </summary>
public ILogger? Logger { get; set; }
public Result<string[]> GetFiles(string path, string searchPattern = "", bool recursive = false)
{
if (IsProtectedPath(ref path))
return Result<string[]>.Fail("Cannot access protected path: " + path);
try
{
return Directory.GetFiles(path, searchPattern ?? string.Empty,
recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
}
catch (Exception)
{
return new string[] { };
}
}
public Result<string[]> GetDirectories(string path)
{
if (IsProtectedPath(ref path))
return Result<string[]>.Fail("Cannot access protected path: " + path);
try
{
return Directory.GetDirectories(path);
}
catch (Exception)
{
return new string[] { };
}
}
public Result<bool> DirectoryExists(string path)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
try
{
return Directory.Exists(path);
}
catch (Exception)
{
return false;
}
}
public Result<bool> DirectoryDelete(string path, bool recursive = false)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
try
{
Directory.Delete(path, recursive);
return true;
}
catch (Exception ex)
{
return Result<bool>.Fail(ex.Message);
}
}
public Result<bool> DirectoryMove(string path, string destination)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
if (IsProtectedPath(ref destination))
return Result<bool>.Fail("Cannot access protected path: " + destination);
try
{
Directory.Move(path, destination);
SetPermissions(destination);
return true;
}
catch (Exception ex)
{
return Result<bool>.Fail(ex.Message);
}
}
public Result<bool> DirectoryCreate(string path)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
try
{
var dirInfo = new DirectoryInfo(path);
if (dirInfo.Exists == false)
dirInfo.Create();
SetPermissions(path);
return true;
}
catch (Exception ex)
{
return Result<bool>.Fail(ex.Message);
}
}
public Result<bool> FileExists(string path)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
try
{
return File.Exists(path);
}
catch (Exception)
{
return false;
}
}
public Result<FileInformation> FileInfo(string path)
{
if (IsProtectedPath(ref path))
return Result<FileInformation>.Fail("Cannot access protected path: " + path);
try
{
FileInfo fileInfo = new FileInfo(path);
return new FileInformation
{
CreationTime = fileInfo.CreationTime,
CreationTimeUtc = fileInfo.CreationTimeUtc,
LastWriteTime = fileInfo.LastWriteTime,
LastWriteTimeUtc = fileInfo.LastWriteTimeUtc,
Extension = fileInfo.Extension.TrimStart('.'),
Name = fileInfo.Name,
FullName = fileInfo.FullName,
Length = fileInfo.Length,
Directory = fileInfo.DirectoryName
};
}
catch (Exception ex)
{
return Result<FileInformation>.Fail(ex.Message);
}
}
public Result<bool> FileDelete(string path)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
try
{
var fileInfo = new FileInfo(path);
if(fileInfo.Exists)
fileInfo.Delete();
return true;
}
catch (Exception)
{
return false;
}
}
public Result<long> FileSize(string path)
{
if (IsProtectedPath(ref path))
return Result<long>.Fail("Cannot access protected path: " + path);
try
{
var fileInfo = new FileInfo(path);
if (fileInfo.Exists == false)
return Result<long>.Fail("File does not exist");
return fileInfo.Length;
}
catch (Exception ex)
{
return Result<long>.Fail(ex.Message);
}
}
public Result<DateTime> FileCreationTimeUtc(string path)
{
if (IsProtectedPath(ref path))
return Result<DateTime>.Fail("Cannot access protected path: " + path);
try
{
var fileInfo = new FileInfo(path);
if (fileInfo.Exists == false)
return Result<DateTime>.Fail("File does not exist");
return fileInfo.CreationTimeUtc;
}
catch (Exception ex)
{
return Result<DateTime>.Fail(ex.Message);
}
}
public Result<DateTime> FileLastWriteTimeUtc(string path)
{
if (IsProtectedPath(ref path))
return Result<DateTime>.Fail("Cannot access protected path: " + path);
try
{
var fileInfo = new FileInfo(path);
if (fileInfo.Exists == false)
return Result<DateTime>.Fail("File does not exist");
return fileInfo.LastWriteTimeUtc;
}
catch (Exception ex)
{
return Result<DateTime>.Fail(ex.Message);
}
}
public Result<bool> FileMove(string path, string destination, bool overwrite = true)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
if (IsProtectedPath(ref destination))
return Result<bool>.Fail("Cannot access protected path: " + destination);
try
{
var fileInfo = new FileInfo(path);
if (fileInfo.Exists == false)
return Result<bool>.Fail("File does not exist");
var destDir = new FileInfo(destination).Directory;
if (destDir.Exists == false)
{
destDir.Create();
SetPermissions(destDir.FullName);
}
fileInfo.MoveTo(destination, overwrite);
SetPermissions(destination);
return true;
}
catch (Exception ex)
{
return Result<bool>.Fail(ex.Message);
}
}
public Result<bool> FileCopy(string path, string destination, bool overwrite = true)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
if (IsProtectedPath(ref destination))
return Result<bool>.Fail("Cannot access protected path: " + destination);
try
{
var fileInfo = new FileInfo(path);
if (fileInfo.Exists == false)
return Result<bool>.Fail("File does not exist");
var destDir = new FileInfo(destination).Directory;
if (destDir.Exists == false)
{
destDir.Create();
SetPermissions(destDir.FullName);
}
fileInfo.CopyTo(destination, overwrite);
SetPermissions(destination);
return true;
}
catch (Exception ex)
{
return Result<bool>.Fail(ex.Message);
}
}
public Result<bool> FileAppendAllText(string path, string text)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
try
{
File.AppendAllText(path, text);
SetPermissions(path);
return true;
}
catch (Exception ex)
{
return Result<bool>.Fail(ex.Message);
}
}
public bool FileIsLocal(string path) => true;
/// <summary>
/// Gets the local path
/// </summary>
/// <param name="path">the path</param>
/// <returns>the local path to the file</returns>
public Result<string> GetLocalPath(string path)
=> Result<string>.Success(path);
public Result<bool> Touch(string path)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
if (DirectoryExists(path).Is(true))
{
try
{
Directory.SetLastWriteTimeUtc(path, DateTime.UtcNow);
return true;
}
catch (Exception ex)
{
return Result<bool>.Fail("Failed to touch directory: " + ex.Message);
}
}
try
{
if (File.Exists(path))
File.SetLastWriteTimeUtc(path, DateTime.UtcNow);
else
{
File.Create(path);
SetPermissions(path);
}
return true;
}
catch (Exception ex)
{
return Result<bool>.Fail($"Failed to touch file: '{path}' => {ex.Message}");
}
}
public Result<bool> SetCreationTimeUtc(string path, DateTime date)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
try
{
if (!File.Exists(path))
return Result<bool>.Fail("File not found.");
File.SetCreationTimeUtc(path, date);
return Result<bool>.Success(true);
}
catch (Exception ex)
{
return Result<bool>.Fail($"Error setting creation time: {ex.Message}");
}
}
public Result<bool> SetLastWriteTimeUtc(string path, DateTime date)
{
if (IsProtectedPath(ref path))
return Result<bool>.Fail("Cannot access protected path: " + path);
try
{
if (!File.Exists(path))
return Result<bool>.Fail("File not found.");
File.SetLastWriteTimeUtc(path, date);
return Result<bool>.Success(true);
}
catch (Exception ex)
{
return Result<bool>.Fail($"Error setting last write time: {ex.Message}");
}
}
/// <summary>
/// Checks if a path is accessible by the file server
/// </summary>
/// <param name="path">the path to check</param>
/// <returns>true if accessible, otherwise false</returns>
private bool IsProtectedPath(ref string path)
{
if (OperatingSystem.IsWindows())
path = path.Replace("/", "\\");
else
path = path.Replace("\\", "/");
if(ReplaceVariables != null)
path = ReplaceVariables(path, true);
if (FileHelper.IsSystemDirectory(path))
return true; // a system directory, no access
if (AllowedPaths?.Any() != true)
return false; // no allowed paths configured, allow all
if (OperatingSystem.IsWindows())
path = path.ToLowerInvariant();
for(int i=0;i<AllowedPaths.Length;i++)
{
string p = OperatingSystem.IsWindows() ? AllowedPaths[i].ToLowerInvariant().TrimEnd('\\') : AllowedPaths[i].TrimEnd('/');
if (path.StartsWith(p))
return false;
}
return true;
}
public void SetPermissions(string path, int? permissions = null, Action<string> logMethod = null)
{
logMethod ??= (string message) => Logger?.ILog(message);
permissions = permissions != null && permissions > 0 ? permissions : Permissions;
if (permissions == null || permissions < 1)
permissions = 777;
if ((File.Exists(path) == false && Directory.Exists(path) == false))
{
logMethod("SetPermissions: File doesnt existing, skipping");
return;
}
//StringLogger stringLogger = new StringLogger();
var logger = new TestLogger();
bool isFile = new FileInfo(path).Exists;
FileHelper.SetPermissions(logger, path, file: isFile,
permissions: permissions.Value.ToString("D3"));
FileHelper.ChangeOwner(logger, path, file: isFile, ownerGroup: OwnerGroup);
logMethod(logger.ToString());
return;
if (OperatingSystem.IsLinux())
{
var filePermissions = FileHelper.ConvertLinuxPermissionsToUnixFileMode(permissions.Value);
if (filePermissions == UnixFileMode.None)
{
logMethod("SetPermissions: Invalid file permissions: " + permissions.Value);
return;
}
File.SetUnixFileMode(path, filePermissions);
logMethod($"SetPermissions: Permission [{filePermissions}] set on file: " + path);
}
}
}
#endif

View File

@@ -66,6 +66,7 @@ public abstract class TestBase
protected string TestFile_MovText_Mp4 => Path.Combine(TestPath, "movtext.mp4");
protected string TestFile_BasicMkv => Path.Combine(TestPath, "basic.mkv");
protected string TestFile_Tag => Path.Combine(TestPath, "tag.mp4");
protected string TestFile_Sitcom => Path.Combine(TestPath, "sitcom.mkv");
protected string TestFile_Pgs => Path.Combine(TestPath, "pgs.mkv");
protected string TestFile_Font => Path.Combine(TestPath, "font.mkv");
protected string TestFile_DefaultSub => Path.Combine(TestPath, "default-sub.mkv");

View File

@@ -65,6 +65,7 @@ namespace FileFlows.VideoNodes
Encoder = new FFMpegEncoder(ffmpegExe, args.Logger);
Encoder.AtTime += AtTimeEvent;
Encoder.OnStatChange += EncoderOnOnStatChange;
if (string.IsNullOrEmpty(outputFile))
outputFile = System.IO.Path.Combine(args.TempPath, Guid.NewGuid() + "." + extension);
@@ -90,6 +91,7 @@ namespace FileFlows.VideoNodes
SetVideoInfo(args, videoInfo, this.Variables ?? new Dictionary<string, object>());
}
Encoder.AtTime -= AtTimeEvent;
Encoder.OnStatChange -= EncoderOnOnStatChange;
Encoder = null;
output = success.output;
return success.successs;
@@ -114,6 +116,9 @@ namespace FileFlows.VideoNodes
Args.PartPercentageUpdate(percent);
}
private void EncoderOnOnStatChange(string name, object value)
=> Args.AdditionalInfoRecorder?.Invoke(name, value, new TimeSpan(0, 1, 0));
public string CheckVideoCodec(string ffmpeg, string vidparams)
{
if (string.IsNullOrEmpty(vidparams))