SDL_image's IMG_Load do not work - sdl-image

I am using IMG_Load() to load png file, but it simply not working.
loadedImage = IMG_Load(filename.c_str()); after this sentence, loadedImage is still NULL,not error happened.
PS:I am using VS C++2008, png file is in the develop folder. Here is my code:(It is exactly what Lazy Foo like)
//The headers
#include "SDL.h"
#include "SDL_image.h"
#include <string>
#pragma comment( linker, "/subsystem:\"windows\" /entry:\"mainCRTStartup\"" )
//Screen attributes
const int SCREEN_WIDTH = 640;
const int SCREEN_HEIGHT = 480;
const int SCREEN_BPP = 32;
//The surfaces
SDL_Surface *image = NULL;
SDL_Surface *screen = NULL;
SDL_Surface *load_image( std::string filename )
{
//The image that's loaded
SDL_Surface* loadedImage = NULL;
//The optimized image that will be used
SDL_Surface* optimizedImage = NULL;
//Load the image using SDL_image
loadedImage = IMG_Load(filename.c_str());
//If the image loaded
if( loadedImage != NULL )
{
//Create an optimized image
//cout<<"Flag";
optimizedImage = SDL_DisplayFormat( loadedImage );
//Free the old image
SDL_FreeSurface( loadedImage );
}
//Return the optimized image
return optimizedImage;
}
void apply_surface( int x, int y, SDL_Surface* source, SDL_Surface* destination )
{
//Rectangle to hold the offsets
SDL_Rect offset;
//Get offsets
offset.x = x;
offset.y = y;
//Blit the surface
SDL_BlitSurface( source, NULL, destination, &offset );
}
bool init()
{
//Initialize all SDL subsystems
if( SDL_Init( SDL_INIT_EVERYTHING ) == -1 )
{
return false;
}
//Set up the screen
screen = SDL_SetVideoMode( SCREEN_WIDTH, SCREEN_HEIGHT, SCREEN_BPP, SDL_SWSURFACE );
//If there was an error in setting up the screen
if( screen == NULL )
{
return false;
}
//Set the window caption
SDL_WM_SetCaption( "PNG test", NULL );
//If everything initialized fine
return true;
}
void clean_up()
{
//Free the surface
SDL_FreeSurface( image );
//Quit SDL
SDL_Quit();
}
int main( int argc, char* args[] )
{
//Initialize
if( init() == false )
{
return 1;
}
//Load the image
image = load_image( "look.png" );
//If there was a problem in loading the image
if( image == NULL )
{
return 5;
}
//Apply the surface to the screen
apply_surface( 0, 0, image, screen );
//Update the screen
if( SDL_Flip( screen ) == -1 )
{
return 1;
}
//Wait 2 seconds
SDL_Delay( 2000 );
//Free the surface and quit SDL
clean_up();
return 0;
}
The output return 5.

my bad.
I just copy SDL_image.dll to exe floder.
I should also copy zlib1.dll and libpng12-0.dll
Actually, all the dll is need, because if there is not such dll, the program won't give any error prompt, which is confusing.

If you download libpng from the website, you will find a file called libpng12.dll. Copying that file into system32 does not work, because SDL_image actually looks for a file called "libpng12-0.dll". I found that out after having implemented the IMG_GetError() method in my code. The error message says it very clearly...
So what you do is copying libpng12.dll to system32 and rename it to libpng12-0.dll! And everything works just fine.

Related

Text dissapears on QOpenGLWidget after undocking

I'm trying to integrate Qt5 and OpenSceneGraph. I've found this minimal example of embedding.
When I encapsulate the widget, defined in the example, into a QDockWidget and add a simple text label to the example, the text dissapears when I undock the container window. When I redock the window the text is shown no more.
I've already ask this on the OpenSceneGraph forum without success. My guess is that it has something to do with the GL related methods and the internal reparenting of the widgets.
In my example only the constructor and the main function are modified from the original.
This is my modified example (main.cpp):
#include<QApplication>
#include<QMainWindow>
#include<QOpenGLWidget>
#include<QMouseEvent>
#include<QWheelEvent>
#include<QDesktopWidget>
#include<QScreen>
#include<QtGlobal>
#include<QWindow>
#include<QWidget>
#include<QDockWidget>
#include<QVBoxLayout>
#include<osg/ref_ptr>
#include<osgViewer/GraphicsWindow>
#include<osgText/Text>
#include<osgViewer/Viewer>
#include<osg/Camera>
#include<osg/ShapeDrawable>
#include<osg/StateSet>
#include<osg/Material>
#include<osgGA/EventQueue>
#include<osgGA/TrackballManipulator>
#include <iostream>
#include <stdio.h>
class QtOSGWidget : public QOpenGLWidget
{
public:
QtOSGWidget(qreal scaleX, qreal scaleY, QWidget* parent = 0)
: QOpenGLWidget(parent)
, _mGraphicsWindow(new osgViewer::GraphicsWindowEmbedded( this->x(), this->y(),
this->width(), this->height() ) )
, _mViewer(new osgViewer::Viewer)
, m_scaleX(scaleX)
, m_scaleY(scaleY)
{
osg::Cylinder* cylinder = new osg::Cylinder( osg::Vec3( 0.f, 0.f, 0.f ), 0.25f, 0.5f );
osg::ShapeDrawable* sd = new osg::ShapeDrawable( cylinder );
sd->setColor( osg::Vec4( 0.8f, 0.5f, 0.2f, 1.f ) );
osg::Geode* geode = new osg::Geode;
geode->addDrawable(sd);
// adding text to the visualization
osgText::Text * test = new osgText::Text();
test->setDataVariance(osg::Object::DYNAMIC);
test->setCharacterSize(1.0);
test->setColor(osg::Vec4(0.0, 0.0, 0.0, 1.0));
test->setAlignment(osgText::Text::CENTER_BOTTOM);
test->setAxisAlignment(osgText::TextBase::SCREEN);
test->setCharacterSizeMode(osgText::Text::OBJECT_COORDS);
test->setText("CYLINDER");
geode->addDrawable(test);
// end adding text
osg::Camera* camera = new osg::Camera;
camera->setViewport( 0, 0, this->width(), this->height() );
camera->setClearColor( osg::Vec4( 0.9f, 0.9f, 1.f, 1.f ) );
float aspectRatio = static_cast<float>( this->width()) / static_cast<float>( this->height() );
camera->setProjectionMatrixAsPerspective( 30.f, aspectRatio, 1.f, 1000.f );
camera->setGraphicsContext( _mGraphicsWindow );
_mViewer->setCamera(camera);
_mViewer->setSceneData(geode);
osgGA::TrackballManipulator* manipulator = new osgGA::TrackballManipulator;
manipulator->setAllowThrow( false );
this->setMouseTracking(true);
_mViewer->setCameraManipulator(manipulator);
_mViewer->setThreadingModel(osgViewer::Viewer::SingleThreaded);
// _mViewer->realize();
}
virtual ~QtOSGWidget(){}
void setScale(qreal X, qreal Y)
{
m_scaleX = X;
m_scaleY = Y;
this->resizeGL(this->width(), this->height());
}
protected:
virtual void paintGL()
{
_mViewer->frame();
}
virtual void resizeGL( int width, int height )
{
this->getEventQueue()->windowResize(this->x()*m_scaleX, this->y() * m_scaleY, width*m_scaleX, height*m_scaleY);
_mGraphicsWindow->resized(this->x()*m_scaleX, this->y() * m_scaleY, width*m_scaleX, height*m_scaleY);
osg::Camera* camera = _mViewer->getCamera();
camera->setViewport(0, 0, this->width()*m_scaleX, this->height()* m_scaleY);
}
virtual void initializeGL()
{
osg::Geode* geode = dynamic_cast<osg::Geode*>(_mViewer->getSceneData());
osg::StateSet* stateSet = geode->getOrCreateStateSet();
osg::Material* material = new osg::Material;
material->setColorMode( osg::Material::AMBIENT_AND_DIFFUSE );
stateSet->setAttributeAndModes( material, osg::StateAttribute::ON );
stateSet->setMode( GL_DEPTH_TEST, osg::StateAttribute::ON );
}
virtual void mouseMoveEvent(QMouseEvent* event)
{
this->getEventQueue()->mouseMotion(event->x()*m_scaleX, event->y()*m_scaleY);
}
virtual void mousePressEvent(QMouseEvent* event)
{
unsigned int button = 0;
switch (event->button())
{
case Qt::LeftButton:
button = 1;
break;
case Qt::MiddleButton:
button = 2;
break;
case Qt::RightButton:
button = 3;
break;
default:
break;
}
this->getEventQueue()->mouseButtonPress(event->x()*m_scaleX, event->y()*m_scaleY, button);
}
virtual void mouseReleaseEvent(QMouseEvent* event)
{
unsigned int button = 0;
switch (event->button())
{
case Qt::LeftButton:
button = 1;
break;
case Qt::MiddleButton:
button = 2;
break;
case Qt::RightButton:
button = 3;
break;
default:
break;
}
this->getEventQueue()->mouseButtonRelease(event->x()*m_scaleX, event->y()*m_scaleY, button);
}
virtual void wheelEvent(QWheelEvent* event)
{
int delta = event->delta();
osgGA::GUIEventAdapter::ScrollingMotion motion = delta > 0 ?
osgGA::GUIEventAdapter::SCROLL_UP : osgGA::GUIEventAdapter::SCROLL_DOWN;
this->getEventQueue()->mouseScroll(motion);
}
virtual bool event(QEvent* event)
{
bool handled = QOpenGLWidget::event(event);
this->update();
return handled;
}
private:
osgGA::EventQueue* getEventQueue() const
{
osgGA::EventQueue* eventQueue = _mGraphicsWindow->getEventQueue();
return eventQueue;
}
osg::ref_ptr<osgViewer::GraphicsWindowEmbedded> _mGraphicsWindow;
osg::ref_ptr<osgViewer::Viewer> _mViewer;
qreal m_scaleX, m_scaleY;
};
int main(int argc, char** argv)
{
// This was necessary for keeping the context when undocking
// without it nothing is shown on the widget when gets undocked
QApplication::setAttribute(Qt::AA_ShareOpenGLContexts);
QApplication qapp(argc, argv);
QMainWindow window;
// Here starts code to put the QtOSGWidget inside a dock area
QDockWidget * visDock = new QDockWidget(&window);
visDock->setFloating(false);
visDock->setFeatures(QDockWidget::DockWidgetFloatable | QDockWidget::DockWidgetMovable);
visDock->setAllowedAreas(Qt::TopDockWidgetArea);
QWidget * dockWidgetContent = new QWidget();
QVBoxLayout * verticalLayout = new QVBoxLayout(dockWidgetContent);
verticalLayout->setSpacing(6);
verticalLayout->setContentsMargins(11, 11, 11, 11);
QWidget * containerWidget = new QWidget(dockWidgetContent);
QHBoxLayout * horizontalLayout = new QHBoxLayout(containerWidget);
horizontalLayout->setSpacing(0);
horizontalLayout->setContentsMargins(11, 11, 11, 11);
horizontalLayout->setContentsMargins(0, 0, 0, 0);
QtOSGWidget* widget = new QtOSGWidget(1, 1, containerWidget);
horizontalLayout->addWidget(widget);
verticalLayout->addWidget(containerWidget);
visDock->setWidget(dockWidgetContent);
window.addDockWidget(static_cast<Qt::DockWidgetArea>(4), visDock);
// No central widget is used for the example
window.setCentralWidget(Q_NULLPTR);
window.show();
return qapp.exec();
}
After _mViewer->setSceneData(geode); add this:
_mGraphicsWindow->init();
initializeGL();
and change the method
virtual void paintGL() {
_mViewer->frame();
}
to
virtual void paintGL() {
_mGraphicsWindow->decrementContextIDUsageCount(_mGraphicsWindow->getMaxContextID());
_mGraphicsWindow->releaseContext();
_mGraphicsWindow->init();
_mViewer->frame();
}
And add thins on main function:
QApplication::setAttribute(Qt::AA_DontCreateNativeWidgetSiblings);
will solve your problem.

Record rtsp stream with ffmpeg in iOS

I've followed iFrameExtractor to successfully stream rtsp in my swift project. In this project, it also has recording function. It basically use avformat_write_header
, av_interleaved_write_frame and av_write_trailer to save the rtsp source into mp4 file.
When I used this project in my device, the rtsp streaming works fine, but recording function will always generate a blank mp4 file with no image and sound.
Could anyone tell me what step that I miss?
I'm using iPhone5 with iOS 9.1 and XCode 7.1.1.
The ffmpeg is 2.8.3 version and followed the compile instruction by CompilationGuide – FFmpeg
Following is the sample code in this project
The function that generate every frame:
-(BOOL)stepFrame {
// AVPacket packet;
int frameFinished=0;
static bool bFirstIFrame=false;
static int64_t vPTS=0, vDTS=0, vAudioPTS=0, vAudioDTS=0;
while(!frameFinished && av_read_frame(pFormatCtx, &packet)>=0) {
// Is this a packet from the video stream?
if(packet.stream_index==videoStream) {
// 20130525 albert.liao modified start
// Initialize a new format context for writing file
if(veVideoRecordState!=eH264RecIdle)
{
switch(veVideoRecordState)
{
case eH264RecInit:
{
if ( !pFormatCtx_Record )
{
int bFlag = 0;
//NSString *videoPath = [NSHomeDirectory() stringByAppendingPathComponent:#"Documents/test.mp4"];
NSString *videoPath = #"/Users/liaokuohsun/iFrameTest.mp4";
const char *file = [videoPath UTF8String];
pFormatCtx_Record = avformat_alloc_context();
bFlag = h264_file_create(file, pFormatCtx_Record, pCodecCtx, pAudioCodecCtx,/*fps*/0.0, packet.data, packet.size );
if(bFlag==true)
{
veVideoRecordState = eH264RecActive;
fprintf(stderr, "h264_file_create success\n");
}
else
{
veVideoRecordState = eH264RecIdle;
fprintf(stderr, "h264_file_create error\n");
}
}
}
//break;
case eH264RecActive:
{
if((bFirstIFrame==false) &&(packet.flags&AV_PKT_FLAG_KEY)==AV_PKT_FLAG_KEY)
{
bFirstIFrame=true;
vPTS = packet.pts ;
vDTS = packet.dts ;
#if 0
NSRunLoop *pRunLoop = [NSRunLoop currentRunLoop];
[pRunLoop addTimer:RecordingTimer forMode:NSDefaultRunLoopMode];
#else
[NSTimer scheduledTimerWithTimeInterval:5.0//2.0
target:self
selector:#selector(StopRecording:)
userInfo:nil
repeats:NO];
#endif
}
// Record audio when 1st i-Frame is obtained
if(bFirstIFrame==true)
{
if ( pFormatCtx_Record )
{
#if PTS_DTS_IS_CORRECT==1
packet.pts = packet.pts - vPTS;
packet.dts = packet.dts - vDTS;
#endif
h264_file_write_frame( pFormatCtx_Record, packet.stream_index, packet.data, packet.size, packet.dts, packet.pts);
}
else
{
NSLog(#"pFormatCtx_Record no exist");
}
}
}
break;
case eH264RecClose:
{
if ( pFormatCtx_Record )
{
h264_file_close(pFormatCtx_Record);
#if 0
// 20130607 Test
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(void)
{
ALAssetsLibrary *library = [[ALAssetsLibrary alloc]init];
NSString *filePathString = [NSHomeDirectory() stringByAppendingPathComponent:#"Documents/test.mp4"];
NSURL *filePathURL = [NSURL fileURLWithPath:filePathString isDirectory:NO];
if(1)// ([library videoAtPathIsCompatibleWithSavedPhotosAlbum:filePathURL])
{
[library writeVideoAtPathToSavedPhotosAlbum:filePathURL completionBlock:^(NSURL *assetURL, NSError *error){
if (error) {
// TODO: error handling
NSLog(#"writeVideoAtPathToSavedPhotosAlbum error");
} else {
// TODO: success handling
NSLog(#"writeVideoAtPathToSavedPhotosAlbum success");
}
}];
}
[library release];
});
#endif
vPTS = 0;
vDTS = 0;
vAudioPTS = 0;
vAudioDTS = 0;
pFormatCtx_Record = NULL;
NSLog(#"h264_file_close() is finished");
}
else
{
NSLog(#"fc no exist");
}
bFirstIFrame = false;
veVideoRecordState = eH264RecIdle;
}
break;
default:
if ( pFormatCtx_Record )
{
h264_file_close(pFormatCtx_Record);
pFormatCtx_Record = NULL;
}
NSLog(#"[ERROR] unexpected veVideoRecordState!!");
veVideoRecordState = eH264RecIdle;
break;
}
}
// Decode video frame
avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);
}
else if(packet.stream_index==audioStream)
{
// 20131024 albert.liao modfied start
static int vPktCount=0;
BOOL bIsAACADTS = FALSE;
int ret = 0;
if(aPlayer.vAACType == eAAC_UNDEFINED)
{
tAACADTSHeaderInfo vxAACADTSHeaderInfo = {0};
bIsAACADTS = [AudioUtilities parseAACADTSHeader:(uint8_t *)packet.data ToHeader:&vxAACADTSHeaderInfo];
}
#synchronized(aPlayer)
{
if(aPlayer==nil)
{
aPlayer = [[AudioPlayer alloc]initAudio:nil withCodecCtx:(AVCodecContext *) pAudioCodecCtx];
NSLog(#"aPlayer initAudio");
if(bIsAACADTS)
{
aPlayer.vAACType = eAAC_ADTS;
//NSLog(#"is ADTS AAC");
}
}
else
{
if(vPktCount<5) // The voice is listened once image is rendered
{
vPktCount++;
}
else
{
if([aPlayer getStatus]!=eAudioRunning)
{
dispatch_async(dispatch_get_main_queue(), ^(void) {
#synchronized(aPlayer)
{
NSLog(#"aPlayer start play");
[aPlayer Play];
}
});
}
}
}
};
#synchronized(aPlayer)
{
int ret = 0;
ret = [aPlayer putAVPacket:&packet];
if(ret <= 0)
NSLog(#"Put Audio Packet Error!!");
}
// 20131024 albert.liao modfied end
if(bFirstIFrame==true)
{
switch(veVideoRecordState)
{
case eH264RecActive:
{
if ( pFormatCtx_Record )
{
h264_file_write_audio_frame(pFormatCtx_Record, pAudioCodecCtx, packet.stream_index, packet.data, packet.size, packet.dts, packet.pts);
}
else
{
NSLog(#"pFormatCtx_Record no exist");
}
}
}
}
}
else
{
//fprintf(stderr, "packet len=%d, Byte=%02X%02X%02X%02X%02X\n",\
packet.size, packet.data[0],packet.data[1],packet.data[2],packet.data[3], packet.data[4]);
}
// 20130525 albert.liao modified end
}
return frameFinished!=0;
}
avformat_write_header:
int h264_file_create(const char *pFilePath, AVFormatContext *fc, AVCodecContext *pCodecCtx,AVCodecContext *pAudioCodecCtx, double fps, void *p, int len )
{
int vRet=0;
AVOutputFormat *of=NULL;
AVStream *pst=NULL;
AVCodecContext *pcc=NULL, *pAudioOutputCodecContext=NULL;
avcodec_register_all();
av_register_all();
av_log_set_level(AV_LOG_VERBOSE);
if(!pFilePath)
{
fprintf(stderr, "FilePath no exist");
return -1;
}
if(!fc)
{
fprintf(stderr, "AVFormatContext no exist");
return -1;
}
fprintf(stderr, "file=%s\n",pFilePath);
// Create container
of = av_guess_format( 0, pFilePath, 0 );
fc->oformat = of;
strcpy( fc->filename, pFilePath );
// Add video stream
pst = avformat_new_stream( fc, 0 );
vVideoStreamIdx = pst->index;
fprintf(stderr,"Video Stream:%d",vVideoStreamIdx);
pcc = pst->codec;
avcodec_get_context_defaults3( pcc, AVMEDIA_TYPE_VIDEO );
// Save the stream as origin setting without convert
pcc->codec_type = pCodecCtx->codec_type;
pcc->codec_id = pCodecCtx->codec_id;
pcc->bit_rate = pCodecCtx->bit_rate;
pcc->width = pCodecCtx->width;
pcc->height = pCodecCtx->height;
if(fps==0)
{
double fps=0.0;
AVRational pTimeBase;
pTimeBase.num = pCodecCtx->time_base.num;
pTimeBase.den = pCodecCtx->time_base.den;
fps = 1.0/ av_q2d(pCodecCtx->time_base)/ FFMAX(pCodecCtx->ticks_per_frame, 1);
fprintf(stderr,"fps_method(tbc): 1/av_q2d()=%g",fps);
pcc->time_base.num = 1;
pcc->time_base.den = fps;
}
else
{
pcc->time_base.num = 1;
pcc->time_base.den = fps;
}
// reference ffmpeg\libavformat\utils.c
// For SPS and PPS in avcC container
pcc->extradata = malloc(sizeof(uint8_t)*pCodecCtx->extradata_size);
memcpy(pcc->extradata, pCodecCtx->extradata, pCodecCtx->extradata_size);
pcc->extradata_size = pCodecCtx->extradata_size;
// For Audio stream
if(pAudioCodecCtx)
{
AVCodec *pAudioCodec=NULL;
AVStream *pst2=NULL;
pAudioCodec = avcodec_find_encoder(AV_CODEC_ID_AAC);
// Add audio stream
pst2 = avformat_new_stream( fc, pAudioCodec );
vAudioStreamIdx = pst2->index;
pAudioOutputCodecContext = pst2->codec;
avcodec_get_context_defaults3( pAudioOutputCodecContext, pAudioCodec );
fprintf(stderr,"Audio Stream:%d",vAudioStreamIdx);
fprintf(stderr,"pAudioCodecCtx->bits_per_coded_sample=%d",pAudioCodecCtx->bits_per_coded_sample);
pAudioOutputCodecContext->codec_type = AVMEDIA_TYPE_AUDIO;
pAudioOutputCodecContext->codec_id = AV_CODEC_ID_AAC;
// Copy the codec attributes
pAudioOutputCodecContext->channels = pAudioCodecCtx->channels;
pAudioOutputCodecContext->channel_layout = pAudioCodecCtx->channel_layout;
pAudioOutputCodecContext->sample_rate = pAudioCodecCtx->sample_rate;
pAudioOutputCodecContext->bit_rate = 12000;//pAudioCodecCtx->sample_rate * pAudioCodecCtx->bits_per_coded_sample;
pAudioOutputCodecContext->bits_per_coded_sample = pAudioCodecCtx->bits_per_coded_sample;
pAudioOutputCodecContext->profile = pAudioCodecCtx->profile;
//FF_PROFILE_AAC_LOW;
// pAudioCodecCtx->bit_rate;
// AV_SAMPLE_FMT_U8P, AV_SAMPLE_FMT_S16P
//pAudioOutputCodecContext->sample_fmt = AV_SAMPLE_FMT_FLTP;//pAudioCodecCtx->sample_fmt;
pAudioOutputCodecContext->sample_fmt = pAudioCodecCtx->sample_fmt;
//pAudioOutputCodecContext->sample_fmt = AV_SAMPLE_FMT_U8;
pAudioOutputCodecContext->sample_aspect_ratio = pAudioCodecCtx->sample_aspect_ratio;
pAudioOutputCodecContext->time_base.num = pAudioCodecCtx->time_base.num;
pAudioOutputCodecContext->time_base.den = pAudioCodecCtx->time_base.den;
pAudioOutputCodecContext->ticks_per_frame = pAudioCodecCtx->ticks_per_frame;
pAudioOutputCodecContext->frame_size = 1024;
fprintf(stderr,"profile:%d, sample_rate:%d, channles:%d", pAudioOutputCodecContext->profile, pAudioOutputCodecContext->sample_rate, pAudioOutputCodecContext->channels);
AVDictionary *opts = NULL;
av_dict_set(&opts, "strict", "experimental", 0);
if (avcodec_open2(pAudioOutputCodecContext, pAudioCodec, &opts) < 0) {
fprintf(stderr, "\ncould not open codec\n");
}
av_dict_free(&opts);
#if 0
// For Audio, this part is no need
if(pAudioCodecCtx->extradata_size!=0)
{
NSLog(#"extradata_size !=0");
pAudioOutputCodecContext->extradata = malloc(sizeof(uint8_t)*pAudioCodecCtx->extradata_size);
memcpy(pAudioOutputCodecContext->extradata, pAudioCodecCtx->extradata, pAudioCodecCtx->extradata_size);
pAudioOutputCodecContext->extradata_size = pAudioCodecCtx->extradata_size;
}
else
{
// For WMA test only
pAudioOutputCodecContext->extradata_size = 0;
NSLog(#"extradata_size ==0");
}
#endif
}
if(fc->oformat->flags & AVFMT_GLOBALHEADER)
{
pcc->flags |= CODEC_FLAG_GLOBAL_HEADER;
pAudioOutputCodecContext->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
if ( !( fc->oformat->flags & AVFMT_NOFILE ) )
{
vRet = avio_open( &fc->pb, fc->filename, AVIO_FLAG_WRITE );
if(vRet!=0)
{
fprintf(stderr,"avio_open(%s) error", fc->filename);
}
}
// dump format in console
av_dump_format(fc, 0, pFilePath, 1);
vRet = avformat_write_header( fc, NULL );
if(vRet==0)
return 1;
else
return 0;
}
av_interleaved_write_frame:
void h264_file_write_frame(AVFormatContext *fc, int vStreamIdx, const void* p, int len, int64_t dts, int64_t pts )
{
AVStream *pst = NULL;
AVPacket pkt;
if ( 0 > vVideoStreamIdx )
return;
// may be audio or video
pst = fc->streams[ vStreamIdx ];
// Init packet
av_init_packet( &pkt );
if(vStreamIdx ==vVideoStreamIdx)
{
pkt.flags |= ( 0 >= getVopType( p, len ) ) ? AV_PKT_FLAG_KEY : 0;
//pkt.flags |= AV_PKT_FLAG_KEY;
pkt.stream_index = pst->index;
pkt.data = (uint8_t*)p;
pkt.size = len;
pkt.dts = AV_NOPTS_VALUE;
pkt.pts = AV_NOPTS_VALUE;
// TODO: mark or unmark the log
//fprintf(stderr, "dts=%lld, pts=%lld\n",dts,pts);
// av_write_frame( fc, &pkt );
}
av_interleaved_write_frame( fc, &pkt );
}
av_write_trailer:
void h264_file_close(AVFormatContext *fc)
{
if ( !fc )
return;
av_write_trailer( fc );
if ( fc->oformat && !( fc->oformat->flags & AVFMT_NOFILE ) && fc->pb )
avio_close( fc->pb );
av_free( fc );
}
Thanks.
It looks like you're using the same AVFormatContext for both the input and output?
In the line
pst = fc->streams[ vStreamIdx ];
You're assigning the AVStream* from your AVFormatContext connected with your input (RTSP stream). But then later on you're trying to write the packet back to the same context av_interleaved_write_frame( fc, &pkt );. I kind of think of a context as a file which has helped me navagate this type of thing better. I do something identicial to what you're doing (not iOS though) where I use a separate AVFormatContext for each of the input (RTSP stream) and output (mp4 file). If I'm correct, I think what you just need to do is initialize an AVFormatContext and properly.
The following code (without error checking everything) is what I do to take an AVFormatContext * output_format_context = NULL and the AVFormatContext * input_format_context that I had associated with the RTSP stream and write from one to the other. This is after I have fetched a packet, etc., which in your case it looks like you're populating (I just take the packet from av_read_frame and re-package it.
This is code that could be in your write frame function (but it also does include the writing of the header).
AVFormatContext * output_format_context;
AVStream * in_stream_2;
AVStream * out_stream_2;
// Allocate the context with the output file
avformat_alloc_output_context2(&output_format_context, NULL, NULL, out_filename.c_str());
// Point to AVOutputFormat * output_format for manipulation
output_format = output_format_context->oformat;
// Loop through all streams
for (i = 0; i < input_format_context->nb_streams; i++) {
// Create a pointer to the input stream that was allocated earlier in the code
AVStream *in_stream = input_format_context->streams[i];
// Create a pointer to a new stream that will be part of the output
AVStream *out_stream = avformat_new_stream(output_format_context, in_stream->codec->codec);
// Set time_base of the new output stream to equal the input stream one since I'm not changing anything (can avoid but get a deprecation warning)
out_stream->time_base = in_stream->time_base;
// This is the non-deprecated way of copying all the parameters from the input stream into the output stream since everything stays the same
avcodec_parameters_from_context(out_stream->codecpar, in_stream->codec);
// I don't remember what this is for :)
out_stream->codec->codec_tag = 0;
// This just sets a flag from the format context to the stream relating to the header
if (output_format_context->oformat->flags & AVFMT_GLOBALHEADER)
out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
// Check NOFILE flag and open the output file context (previously the output file was associated with the format context, now it is actually opened.
if (!(output_format->flags & AVFMT_NOFILE))
avio_open(&output_format_context->pb, out_filename.c_str(), AVIO_FLAG_WRITE);
// Write the header (not sure if this is always needed but h264 I believe it is.
avformat_write_header(output_format_context,NULL);
// Re-getting the appropriate stream that was populated above (this should allow for both audio/video)
in_stream_2 = input_format_context->streams[packet.stream_index];
out_stream_2 = output_format_context->streams[packet.stream_index];
// Rescaling pts and dts, duration and pos - you would do as you need in your code.
packet.pts = av_rescale_q_rnd(packet.pts, in_stream_2->time_base, out_stream_2->time_base, (AVRounding) (AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
packet.dts = av_rescale_q_rnd(packet.dts, in_stream_2->time_base, out_stream_2->time_base, (AVRounding) (AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
packet.duration = av_rescale_q(packet.duration, in_stream_2->time_base, out_stream_2->time_base);
packet.pos = -1;
// The first packet of my stream always gives me negative dts/pts so this just protects that first one for my purposes. You probably don't need.
if (packet.dts < 0) packet.dts = 0;
if (packet.pts < 0) packet.pts = 0;
// Finally write the frame
av_interleaved_write_frame(output_format_context, &packet);
// ....
// Write header, close/cleanup... etc
// ....
This code is fairly bare bones and doesn't include the setup (which it sounds like you're doing correctly anyway). I would also imagine this code could be cleaned up and tweaked for your purposes, but this works for me to re-write the RTSP stream into a file (in my case many files but code not shown).
The code is C code, so you might need to do minor tweaks for making it Swift compatible (for some of the library function calls maybe). I think overall it should be compatible though.
Hopefully this helps point you to the right direction. This was cobbled together thanks to several sample code sources (I don't remember where), along with warning prompts from the libraries themselves.

How to acquire skeletal joint data to text file from two kinect cameras in SkeletalViewer? (C++)

I am currently working on a project to use multiple kinect cameras to acquire x,y,z coordinates of skeletal data in SkeletalViewer. I have an idea to use the KinectID or index of different kinect cameras to extract the sets of skeletal joints data into 2 different text files. But I am not sure if I am doing right. Please help to take a look at the modification below, or I will appreciate all your kind advice on other method to solve this problem.
In SkeletalViewer.h, I modified as following:
public:
FILE* mp3DFile0;
FILE* mp3DFile1;
char mText[1024];
INuiSensor * m_pNuiSensor;
BSTR m_instanceId;
array SensorIndex;
In NuiImpl.cpp, I modified as following:
1) define array
ref class MyClass {
public:
int m_i;
};
array<MyClass^>^ arrSensor() {
int i;
array< MyClass^ >^ local = gcnew array< MyClass^ >(2);
for (i = 0; i < 2; i++) {
local[i] = gcnew MyClass;
local[i]->m_i = i;
}
return local;
}
2) create array to store sensor index to do for loop later
HRESULT CSkeletalViewerApp::Nui_Init( )
{
HRESULT hr;
bool result;
//create an array to store two file pointers
FILE* mp3DFile[] = { mp3DFile0, mp3DFile1 };
fopen_s(mp3DFile0, "D:/Kinect/KinectCam0.txt", "w+");
fopen_s(mp3DFile1, "D:/Kinect/KinectCam1.txt", "w+");
.
.
if ( !m_pNuiSensor )
{
hr = NuiCreateSensorByIndex(0, &m_pNuiSensor);
//I am not sure about how to utilize this index in this case
.
.
}
if (NuiGetSensorCount(&m_pNuiSensor) > 1)
{
array< MyClass^ >^ SensorIndex;
SensorIndex = arrSensor();
}
}
3) use for loop to store data to different text file using index
void CSkeletalViewerApp::Nui_DrawSkeleton( const NUI_SKELETON_DATA & skel, int windowWidth, int windowHeight )
{
int i;
int h;
for (h = 0; h < 2; h++)
{
//when index point to the current kinect
if (SensorIndex[h] == &m_pNuiSensor)
{
for (i = 0; i < NUI_SKELETON_POSITION_COUNT; i++)
{
m_Points[i] = SkeletonToScreen(skel.SkeletonPositions[i], windowWidth, windowHeight);
memset(mText, 0, 1024);
sprintf(mText, "(%0.3f,%0.3f,%0.3f)", skel.SkeletonPositions[i].x, skel.SkeletonPositions[i].y, skel.SkeletonPositions[i].z);
if (mp3DFile[h]) {
fputs((const char*)mText, mp3DFile[h]);
}
}
if (mp3DFile[h]) {
fputs("\n", mp3DFile[h]);
}
}
}
.
.
}
I am a newbie in this Kinect programming. Thank you very much for your help! :)

How to write NALs produced by x264_encoder_encode() using ffmpeg av_interleaved_write_frame()

I have been trying to produce a "flv" video file in the following sequence:
av_register_all();
// Open video file
if (avformat_open_input(&pFormatCtx, "6.mp4", NULL, NULL) != 0)
return -1; // Couldn't open file
// Retrieve stream information
if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
return -1; // Couldn't find stream information
// Dump information about file onto standard error
av_dump_format(pFormatCtx, 0, "input_file.mp4", 0);
// Find the first video stream
videoStream = -1;
for (i = 0; i < pFormatCtx->nb_streams; i++)
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
videoStream = i;
break;
}
if (videoStream == -1)
return -1; // Didn't find a video stream
// Get a pointer to the codec context for the video stream
pCodecCtx = pFormatCtx->streams[videoStream]->codec;
// Find the decoder for the video stream
pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if (pCodec == NULL) {
fprintf(stderr, "Unsupported codec!\n");
return -1; // Codec not found
}
// Open codec
if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
return -1; // Could not open codec
// Allocate video frame
pFrame = avcodec_alloc_frame();
// Allocate video frame
pFrame = avcodec_alloc_frame();
// Allocate an AVFrame structure
pFrameYUV420 = avcodec_alloc_frame();
if (pFrameYUV420 == NULL)
return -1;
// Determine required buffer size and allocate buffer
numBytes = avpicture_get_size(pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height);
buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
// Assign appropriate parts of buffer to image planes in pFrameYUV420
// Note that pFrameYUV420 is an AVFrame, but AVFrame is a superset of AVPicture
avpicture_fill((AVPicture *) pFrameRGB, buffer, pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height);
// Setup scaler
img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt, SWS_BILINEAR, 0, 0, 0);
if (img_convert_ctx == NULL) {
fprintf(stderr, "Cannot initialize the conversion context!\n");
exit(1);
}
// Setup encoder/muxing now
filename = "output_file.flv";
fmt = av_guess_format("flv", filename, NULL);
if (fmt == NULL) {
printf("Could not guess format.\n");
return -1;
}
/* allocate the output media context */
oc = avformat_alloc_context();
if (oc == NULL) {
printf("could not allocate context.\n");
return -1;
}
oc->oformat = fmt;
snprintf(oc->filename, sizeof(oc->filename), "%s", filename);
video_st = NULL;
if (fmt->video_codec != AV_CODEC_ID_NONE) {
video_st = add_stream(oc, &video_codec, fmt->video_codec);
}
// Let's see some information about our format
av_dump_format(oc, 0, filename, 1);
/* open the output file, if needed */
if (!(fmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&oc->pb, filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open '%s': %s\n", filename, av_err2str(ret));
return 1;
}
}
/* Write the stream header, if any. */
ret = avformat_write_header(oc, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file: %s\n", av_err2str(ret));
return 1;
}
// Setup x264 params
x264_param_t param;
x264_param_default_preset(&param, "veryfast", "zerolatency");
param.i_threads = 1;
param.i_width = video_st->codec->width;
param.i_height = video_st->codec->height;
param.i_fps_num = STREAM_FRAME_RATE; // 30 fps, same as video
param.i_fps_den = 1;
// Intra refres:
param.i_keyint_max = STREAM_FRAME_RATE;
param.b_intra_refresh = 1;
// Rate control:
param.rc.i_rc_method = X264_RC_CRF;
param.rc.f_rf_constant = 25;
param.rc.f_rf_constant_max = 35;
// For streaming:
param.b_repeat_headers = 1;
param.b_annexb = 1;
x264_param_apply_profile(&param, "baseline");
x264_t* encoder = x264_encoder_open(&param);
x264_picture_t pic_in, pic_out;
x264_picture_alloc(&pic_in, X264_CSP_I420, video_st->codec->width, video_st->codec->height);
x264_nal_t* nals;
int i_nals;
// The loop:
// 1. Read frames
// 2. Decode the frame
// 3. Attempt to re-encode using x264
// 4. Write the x264 encoded frame using av_interleaved_write_frame
while (av_read_frame(pFormatCtx, &packet) >= 0) {
// Is this a packet from the video stream?
if (packet.stream_index == videoStream) {
// Decode video frame
avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet);
// Did we get a video frame?
if (frameFinished) {
sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0, pCodecCtx->height, pic_in.img.plane, pic_in.img.i_stride);
int frame_size = x264_encoder_encode(encoder, &nals, &i_nals, &pic_in, &pic_out);
if (frame_size >= 0) {
if (i_nals < 0)
printf("invalid frame size: %d\n", i_nals);
// write out NALs
for (i = 0; i < i_nals; i++) {
// initalize a packet
AVPacket p;
av_init_packet(&p);
p.data = nals[i].p_payload;
p.size = nals[i].i_payload;
p.stream_index = video_st->index;
p.flags = AV_PKT_FLAG_KEY;
p.pts = AV_NOPTS_VALUE;
p.dts = AV_NOPTS_VALUE;
ret = av_interleaved_write_frame(oc, &p);
}
}
printf("encoded frame #%d\n", frame_count);
frame_count++;
}
}
// Free the packet that was allocated by av_read_frame
av_free_packet(&packet);
}
// Now we free up resources used/close codecs, and finally close our program.
Here is the implementation for the add_stream() function:
/* Add an output stream. */
static AVStream *add_stream(AVFormatContext *oc, AVCodec **codec, enum AVCodecID codec_id) {
AVCodecContext *c;
AVStream *st;
int r;
/* find the encoder */
*codec = avcodec_find_encoder(codec_id);
if (!(*codec)) {
fprintf(stderr, "Could not find encoder for '%s'\n",
avcodec_get_name(codec_id));
exit(1);
}
st = avformat_new_stream(oc, *codec);
if (!st) {
fprintf(stderr, "Could not allocate stream\n");
exit(1);
}
st->id = oc->nb_streams - 1;
c = st->codec;
switch ((*codec)->type) {
case AVMEDIA_TYPE_AUDIO:
st->id = 1;
c->sample_fmt = AV_SAMPLE_FMT_FLTP;
c->bit_rate = 64000;
c->sample_rate = 44100;
c->channels = 2;
break;
case AVMEDIA_TYPE_VIDEO:
avcodec_get_context_defaults3(c, *codec);
c->codec_id = codec_id;
c->bit_rate = 500*1000;
//c->rc_min_rate = 500*1000;
//c->rc_max_rate = 500*1000;
//c->rc_buffer_size = 500*1000;
/* Resolution must be a multiple of two. */
c->width = 1280;
c->height = 720;
/* timebase: This is the fundamental unit of time (in seconds) in terms
* of which frame timestamps are represented. For fixed-fps content,
* timebase should be 1/framerate and timestamp increments should be
* identical to 1. */
c->time_base.den = STREAM_FRAME_RATE;
c->time_base.num = 1;
c->gop_size = 12; /* emit one intra frame every twelve frames at most */
c->pix_fmt = STREAM_PIX_FMT;
if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
/* just for testing, we also add B frames */
c->max_b_frames = 2;
}
if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) {
/* Needed to avoid using macroblocks in which some coeffs overflow.
* This does not happen with normal video, it just happens here as
* the motion of the chroma plane does not match the luma plane. */
c->mb_decision = 2;
}
break;
default:
break;
}
/* Some formats want stream headers to be separate. */
if (oc->oformat->flags & AVFMT_GLOBALHEADER)
c->flags |= CODEC_FLAG_GLOBAL_HEADER;
return st;
}
After the encoding is complete, I check the output file output_file.flv. I notice it's size is very large: 101MB and it does not play. If I use ffmpeg to decode/encode the input file, then I get an output file about 83MB in size (which is about the same size as the original .mp4 file used as input). Also, the 83MB output from just using ffmpeg C api, as opposed to using x264 for the encoding step, plays just fine. Does anyone know where I am going wrong? I have tried researching this for a few days now but with no luck :(. I feel that I am close to making it work, however, I just cannot figure out what I am doing wrong. Thank you!
To produce the correct AVPacket, you should write all nals into the same packet, as it is done in http://ffmpeg.org/doxygen/trunk/libx264_8c_source.html (see encode_nals and X264_frame functions)

How to badge file and folder using Cocoa

I want to badge a file and folder with some color (image). How can this be achieved?
I tried with icon service, and it works for files, but it is not working with folders.
I saw this behavior working Dropbox (10.4, 10.5 and 10.6)- how can this be done?
The blog post Cocoa Tutorial: Custom Folder Icons was very close one for me, but it was not working as expected.
Is there another solution other than icon service?
The following function is the solution I found for the problem
BOOL AddBadgeToItem(NSString* path,NSData* tag)
{
FSCatalogInfo info;
FSRef par;
FSRef ref;
Boolean dir = false;
if (tag&&(FSPathMakeRef([path fileSystemRepresentation],&par,&dir)==noErr))
{
HFSUniStr255 fork = {0,{0}};
sint16 refnum = kResFileNotOpened;
FSGetResourceForkName(&fork);
if (dir)
{
NSString *name = #"Icon\r";
memset(&info,0,sizeof(info));
((FileInfo*)(&info.finderInfo))->finderFlags = kIsInvisible;
OSErr error = FSCreateResourceFile(&par,[name lengthOfBytesUsingEncoding:NSUTF16LittleEndianStringEncoding],(UniChar*)[name cStringUsingEncoding:NSUTF16LittleEndianStringEncoding],kFSCatInfoFinderXInfo,&info,fork.length, fork.unicode,&ref,NULL);
if( error == dupFNErr )
{
// file already exists; prepare to try to open it
const char *iconFileSystemPath = [[path stringByAppendingPathComponent:#"\000I\000c\000o\000n\000\r"] fileSystemRepresentation];
OSStatus status = FSPathMakeRef((const UInt8 *)iconFileSystemPath, &ref, NULL);
if (status != noErr)
{
fprintf(stderr, "error: FSPathMakeRef() returned %d for file \"%s\"\n", (int)status, iconFileSystemPath);
}
}else if ( error != noErr)
{
return NO;
}
}
else
{
BlockMoveData(&par,&ref,sizeof(FSRef));
if (FSCreateResourceFork(&ref,fork.length,fork.unicode,0)!=noErr)
{
//test
if (FSOpenResourceFile(&ref,fork.length,fork.unicode,fsRdWrPerm,&refnum)!=noErr) {
return NO;
}
if (refnum!=kResFileNotOpened) {
UpdateResFile(refnum);
CloseResFile(refnum);
if (FSGetCatalogInfo(&par,kFSCatInfoFinderXInfo,&info,NULL,NULL,NULL)==noErr) {
((ExtendedFileInfo*)(&info.extFinderInfo))->extendedFinderFlags = kExtendedFlagsAreInvalid;
FSSetCatalogInfo(&par,kFSCatInfoFinderXInfo,&info);
}
}
//Test end
return NO;
}
}
OSErr errorr = FSOpenResourceFile(&ref,fork.length,fork.unicode,fsRdWrPerm,&refnum);
if (errorr!=noErr) {
return NO;
}
if (refnum!=kResFileNotOpened) {
CustomBadgeResource* cbr;
int len = [tag length];
Handle h = NewHandle(len);
if (h) {
BlockMoveData([tag bytes],*h,len);
AddResource(h,kIconFamilyType,128,"\p");
WriteResource(h);
ReleaseResource(h);
}
h = NewHandle(sizeof(CustomBadgeResource));
if (h) {
cbr = (CustomBadgeResource*)*h;
memset(cbr,0,sizeof(CustomBadgeResource));
cbr->version = kCustomBadgeResourceVersion;
cbr->customBadgeResourceID = 128;
AddResource(h,kCustomBadgeResourceType,kCustomBadgeResourceID,"\p");
WriteResource(h);
ReleaseResource(h);
}
UpdateResFile(refnum);
CloseResFile(refnum);
if (FSGetCatalogInfo(&par,kFSCatInfoFinderXInfo,&info,NULL,NULL,NULL)==noErr) {
((ExtendedFileInfo*)(&info.extFinderInfo))->extendedFinderFlags = kExtendedFlagHasCustomBadge;
FSSetCatalogInfo(&par,kFSCatInfoFinderXInfo,&info);
}
}
}
return NO;
}
You can do this using the -setIcon:forFile:options: method on NSWorkspace, which lets you just specify an NSImage to apply to the file/folder at the path you give it.
The intended approach is to create a Finder Sync app extension.