mirror/OpenSpace
1
0
Fork 0
mirror of https://github.com/OpenSpace/OpenSpace.git synced 2026-03-11 15:58:45 -05:00
Code Issues Packages Projects Releases Wiki Activity

Make sure that the loading of video works and add test function to print the individual frames as images to files

Browse Source
This commit is contained in:
Ylva Selling
2022-09-09 07:14:19 -04:00
parent 406d58a02a
commit ae1dd8aa52
2 changed files with 123 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

329
modules/globebrowsing/src/tileprovider/ffmpegtileprovider.cpp
View File

@@ -27,6 +27,7 @@
#include <openspace/documentation/documentation.h>
#include <openspace/util/time.h>
#include <ghoul/filesystem/filesystem.h>
#include <iostream>
namespace {
constexpr std::string_view _loggerCat = "FfmpegTileProvider";
@@ -58,6 +59,21 @@ namespace {
namespace openspace::globebrowsing {
void save_gray_frame(unsigned char* buf, int wrap, int xsize, int ysize, const char* filename)
{
FILE* f;
int i;
f = fopen(filename, "w");
// writing the minimal required header for a pgm file format
// portable graymap format -> https://en.wikipedia.org/wiki/Netpbm_format#PGM_example
fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
// writing line by line
for (i = 0; i < ysize; i++)
fwrite(buf + i * wrap, 1, xsize, f);
fclose(f);
}
documentation::Documentation FfmpegTileProvider::Documentation() {
return codegen::doc<Parameters>("globebrowsing_ffmpegtileprovider");
}
@@ -73,15 +89,86 @@ FfmpegTileProvider::FfmpegTileProvider(const ghoul::Dictionary& dictionary)
_startTime = p.startTime;
reset();
// Allocate the video frames
_packet = av_packet_alloc();
_avFrame = av_frame_alloc();
_glFrame = av_frame_alloc();
std::string path = absPath(_videoFile).string();
// Open video
int openRes = avformat_open_input(
&_formatContext,
path.c_str(),
nullptr,
nullptr
);
if (openRes < 0) {
LERRORC("FfmpegTileProvider", "Failed to open input for file " + path);
return;
}
// Find stream info
if (avformat_find_stream_info(_formatContext, nullptr) < 0) {
LERRORC("FfmpegTileProvider", "Failed to get stream info for " + path);
return;
}
// Dump debug info
av_dump_format(_formatContext, 0, path.c_str(), false);
for (unsigned int i = 0; i < _formatContext->nb_streams; ++i) {
AVMediaType codec = _formatContext->streams[i]->codecpar->codec_type;
if (codec == AVMEDIA_TYPE_VIDEO) {
_streamIndex = i;
_videoStream = _formatContext->streams[_streamIndex];
break;
}
}
if (_streamIndex == -1 || _videoStream == nullptr) {
LERRORC("FfmpegTileProvider", "Failed to find video stream for " + path);
return;
}
// Find decoder
_decoder = avcodec_find_decoder(_videoStream->codecpar->codec_id);
if (!_decoder) {
LERRORC("FfmpegTileProvider", "Failed to find decoder for " + path);
return;
}
_codecContext = avcodec_alloc_context3(nullptr);
int contextSuccess = avcodec_parameters_to_context(
_codecContext,
_videoStream->codecpar
);
if (contextSuccess < 0) {
LERRORC(
"FfmpegTileProvider",
"Failed to create codec context for " + path
);
return;
}
_nativeSize = { _codecContext->width, _codecContext->height };
// Open the decoder
if (avcodec_open2(_codecContext, _decoder, nullptr) < 0) {
LERRORC("FfmpegTileProvider", "Failed to open codec for " + path);
return;
}
_lastFrameTime = std::max(Time::convertTime(_startTime), Time::now().j2000Seconds());
}
Tile FfmpegTileProvider::tile(const TileIndex& tileIndex) {
ZoneScoped
return Tile();
return _tile;
}
Tile::Status FfmpegTileProvider::tileStatus(const TileIndex&) {
return Tile::Status::OK;
return _tile.status;
}
TileDepthTransform FfmpegTileProvider::depthTransform() {
@@ -97,19 +184,16 @@ void FfmpegTileProvider::update() {
const bool hasNewFrame = (now > Time::convertTime(_startTime)) &&
(now - _lastFrameTime) > _frameTime;
if (!hasNewFrame) {
return;
if(!hasNewFrame) {
//return; wait with this for now
}
// Read frame
do {
if (!_formatContext || !_packet) {
break;
}
while(true) {
int result = av_read_frame(_formatContext, _packet);
if (result < 0) {
av_packet_unref(_packet);
break;
return;
}
// Does this packet belong to this video stream?
@@ -119,10 +203,10 @@ void FfmpegTileProvider::update() {
// Send packet to the decoder
result = avcodec_send_packet(_codecContext, _packet);
if (result < 0) {
if (result < 0 || result == AVERROR(EAGAIN) || result == AVERROR_EOF) {
LERROR(fmt::format("Sending packet failed with {}", result));
av_packet_unref(_packet);
break;
return;
}
// Get result from decoder
@@ -139,105 +223,21 @@ void FfmpegTileProvider::update() {
if (result < 0) {
LERROR(fmt::format("Receiving packet failed with {}", result));
av_packet_unref(_packet);
break;
return;
}
// We have a new full frame!
_lastFrameTime = now;
// @TODO Do something with it!
_glFrame = av_frame_alloc();
int sz = av_image_get_buffer_size(
AV_PIX_FMT_RGB24,
_codecContext->width,
_codecContext->height,
1
);
uint8_t* internalBuffer = reinterpret_cast<uint8_t*>(av_malloc(sz * sizeof(uint8_t)));
av_image_fill_arrays(
_glFrame->data,
_glFrame->linesize,
internalBuffer,
AV_PIX_FMT_RGB24,
_codecContext->width,
_codecContext->height,
1
);
_packet = av_packet_alloc();
glGenTextures(1, &_frameTexture);
glBindTexture(GL_TEXTURE_2D, _frameTexture);
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexImage2D(
GL_TEXTURE_2D,
0,
GL_RGB,
_codecContext->width,
_codecContext->height,
0,
GL_RGB,
GL_UNSIGNED_BYTE,
nullptr
);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glPushAttrib(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glEnable(GL_TEXTURE_2D);
glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_DECAL);
glBindTexture(GL_TEXTURE_2D, _frameTexture);
glFlush();
glDisable(GL_TEXTURE_2D);
///FBO
glGenFramebuffers(1, &_FBO);
glBindFramebuffer(GL_FRAMEBUFFER, _FBO);
//Attach 2D texture to this FBO
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, _frameTexture, 0);
// Print to file
FILE* output_image;
int output_width, output_height;
output_width = _codecContext->width,
output_height = _codecContext->height;
/// READ THE PIXELS VALUES from FBO AND SAVE TO A .PPM FILE
int i, j, k;
unsigned char* pixels = (unsigned char*)malloc(output_width * output_height * 3);
/// READ THE CONTENT FROM THE FBO
glReadBuffer(GL_COLOR_ATTACHMENT0);
glReadPixels(0, 0, output_width, output_height, GL_RGB, GL_UNSIGNED_BYTE, pixels);
output_image = fopen("C:\\Users\\ylvaselling\\Documents\\Work\\Dataset\\output.ppm", "wt");
fprintf(output_image, "P3\n");
fprintf(output_image, "# Created by Ricao\n");
fprintf(output_image, "%d %d\n", output_width, output_height);
fprintf(output_image, "255\n");
k = 0;
for (i = 0; i < output_width; i++)
{
for (j = 0; j < output_height; j++)
{
fprintf(output_image, "%u %u %u ", (unsigned int)pixels[k], (unsigned int)pixels[k + 1],
(unsigned int)pixels[k + 2]);
k = k + 3;
}
fprintf(output_image, "\n");
}
free(pixels);
// Successfully collected a frame
std::cout << "frame: " << _codecContext->frame_number << std::endl;
break;
}
av_packet_unref(_packet);
} while (true);
// TEST save a grayscale frame into a .pgm file
// This ends up in OpenSpace\build\apps\OpenSpace
// https://github.com/leandromoreira/ffmpeg-libav-tutorial/blob/master/0_hello_world.c
char frame_filename[1024];
snprintf(frame_filename, sizeof(frame_filename), "%s-%d.pgm", "frame", _codecContext->frame_number);
save_gray_frame(_avFrame->data[0], _avFrame->linesize[0], _avFrame->width, _avFrame->height, frame_filename);
av_packet_unref(_packet);
}
int FfmpegTileProvider::minLevel() {
@@ -256,114 +256,13 @@ float FfmpegTileProvider::noDataValueAsFloat() {
}
void FfmpegTileProvider::internalInitialize() {
std::string path = absPath(_videoFile).string();
int result;
// Open the video
result = avformat_open_input(
&_formatContext,
path.c_str(),
nullptr,
nullptr
);
if (result < 0) {
LERROR(fmt::format("Failed to open input for video file {}", _videoFile));
return;
}
// Get stream info
if (avformat_find_stream_info(_formatContext, nullptr) < 0) {
LERROR(fmt::format("Failed to get stream info for {}", _videoFile));
return;
}
// Debug
//av_dump_format(_formatContext, 0, path.c_str(), 0);
// Find the stream with the video (there anc also be audio and data streams)
for (unsigned int i = 0; i < _formatContext->nb_streams; ++i) {
AVMediaType codec = _formatContext->streams[i]->codecpar->codec_type;
if (codec == AVMEDIA_TYPE_VIDEO) {
_streamIndex = i;
break;
}
}
if (_streamIndex == -1) {
LERROR(fmt::format("Failed to find video stream for {}", _videoFile));
return;
}
_videoStream = _formatContext->streams[_streamIndex];
_codecContext = avcodec_alloc_context3(nullptr);
result = avcodec_parameters_to_context(
_codecContext,
_videoStream->codecpar
);
if (result) {
LERROR(fmt::format("Failed to create codec context for {}", _videoFile));
return;
}
// Get the size of the video
_nativeSize = glm::ivec2(_codecContext->width, _codecContext->height);
// Get the decoder
_decoder = avcodec_find_decoder(_codecContext->codec_id);
if (!_decoder) {
LERROR(fmt::format("Failed to find decoder for {}", _videoFile));
return;
}
// Open the decoder
result = avcodec_open2(_codecContext, _decoder, nullptr);
if (result < 0) {
LERROR(fmt::format("Failed to open codec for {}", _videoFile));
return;
}
// Allocate the video frames
_avFrame = av_frame_alloc();
_glFrame = av_frame_alloc();
int bufferSize = av_image_get_buffer_size(
AV_PIX_FMT_RGB24,
_codecContext->width,
_codecContext->height,
1
);
uint8_t* internalBuffer =
reinterpret_cast<uint8_t*>(av_malloc(bufferSize * sizeof(uint8_t)));
result = av_image_fill_arrays(
_glFrame->data,
_glFrame->linesize,
internalBuffer,
AV_PIX_FMT_RGB24,
_codecContext->width,
_codecContext->height,
1
);
if (result < 0) {
LERROR(fmt::format("Failed to fill buffer data for video {}", _videoFile));
return;
}
// Allocate packet
_packet = av_packet_alloc();
// Read the first frame to get the framerate of the video
if (_formatContext && _codecContext && _packet) {
av_read_frame(_formatContext, _packet);
avcodec_send_packet(_codecContext, _packet);
_frameTime = av_q2d(_codecContext->time_base) * _codecContext->ticks_per_frame;
}
else {
LERROR(fmt::format("Error loading video {}", path));
}
_lastFrameTime = std::max(Time::convertTime(_startTime), Time::now().j2000Seconds());
// TODO: Currently the update function is called before this
// function - fix that and then move constructor code here
}
void FfmpegTileProvider::internalDeinitialize() {
FfmpegTileProvider::~FfmpegTileProvider() {
// TODO: Check so internalDeinitialize is called after the last
// update function and move code there
avformat_close_input(&_formatContext);
av_free(_avFrame);
av_free(_glFrame);
@@ -371,4 +270,8 @@ void FfmpegTileProvider::internalDeinitialize() {
avformat_free_context(_formatContext);
}
void FfmpegTileProvider::internalDeinitialize() {
}
} // namespace openspace::globebrowsing

18
modules/globebrowsing/src/tileprovider/ffmpegtileprovider.h
View File

@@ -31,12 +31,8 @@
// FFMPEG
extern "C" {
#include <libavcodec/avcodec.h> // avcodec_alloc_context3
#include <libavformat/avformat.h> // avformat_open_input, AVFormatContext
#include <libavutil/imgutils.h> // av_image_get_buffer_size
#include <libavutil/frame.h>
#include <libavutil/mem.h>
}
namespace openspace { struct Documentation; }
@@ -45,7 +41,8 @@ namespace openspace::globebrowsing {
class FfmpegTileProvider : public TileProvider {
public:
FfmpegTileProvider(const ghoul::Dictionary& dictionary);
FfmpegTileProvider(const ghoul::Dictionary& dictionary);
~FfmpegTileProvider();
Tile tile(const TileIndex& tileIndex) override final;
Tile::Status tileStatus(const TileIndex& index) override final;
@@ -65,19 +62,18 @@ private:
double _lastFrameTime; // The in gmae time of the last frame in J2000 seconds
std::string _startTime;
GLuint _frameTexture;
GLuint _FBO = 1; ///Frame-buffer Object
AVFormatContext* _formatContext = nullptr;
int _streamIndex = -1;
AVStream* _videoStream = nullptr;
AVCodecContext* _codecContext = nullptr;
const AVCodec* _decoder = nullptr;
AVFrame* _avFrame = nullptr;
AVFrame* _glFrame = nullptr;
int _streamIndex = -1;
AVStream* _videoStream = nullptr;
AVPacket* _packet = nullptr;
std::unique_ptr<ghoul::opengl::Texture> _tileTexture;
Tile _tile;
void internalInitialize() override final;
void internalDeinitialize() override final;
};