From 86d8f5232b3da5d8d695aaa385ee65b5480b73dc Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 16:31:32 -0700 Subject: [PATCH 01/19] Large-scale s/QT/AV/ --- macos/camera.h | 24 +++++++++++----------- macos/camera.m | 55 +++++++++++++++++++++++++------------------------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/macos/camera.h b/macos/camera.h index 1c6fc02..d53c12a 100644 --- a/macos/camera.h +++ b/macos/camera.h @@ -3,7 +3,7 @@ // #import -#import +#import #define error(...) fprintf(stderr, __VA_ARGS__) #define console(...) (!g_quiet && printf(__VA_ARGS__)) @@ -14,32 +14,32 @@ BOOL g_quiet = NO; @interface ImageSnap : NSObject { - QTCaptureSession *mCaptureSession; - QTCaptureDeviceInput *mCaptureDeviceInput; - QTCaptureDecompressedVideoOutput *mCaptureDecompressedVideoOutput; + AVCaptureSession *mCaptureSession; + AVCaptureDeviceInput *mCaptureDeviceInput; + AVCaptureVideoDataOutput *mCaptureDecompressedVideoOutput; CVImageBufferRef mCurrentImageBuffer; } /** - * Returns all attached QTCaptureDevice objects that have video. - * This includes video-only devices (QTMediaTypeVideo) and - * audio/video devices (QTMediaTypeMuxed). + * Returns all attached AVCaptureDevice objects that have video. + * This includes video-only devices (AVMediaTypeVideo) and + * audio/video devices (AVMediaTypeMuxed). * * @return autoreleased array of video devices */ +(NSArray *)videoDevices; /** - * Returns the default QTCaptureDevice object for video + * Returns the default AVCaptureDevice object for video * or nil if none is found. */ -+(QTCaptureDevice *)defaultVideoDevice; ++(AVCaptureDevice *)defaultVideoDevice; /** - * Returns the QTCaptureDevice with the given name + * Returns the AVCaptureDevice with the given name * or nil if the device cannot be found. */ -+(QTCaptureDevice *)deviceNamed:(NSString *)name; ++(AVCaptureDevice *)deviceNamed:(NSString *)name; /** * Writes an NSImage to disk, formatting it according @@ -58,7 +58,7 @@ BOOL g_quiet = NO; -(id)init; -(void)dealloc; --(BOOL)startSession:(QTCaptureDevice *)device withWidth:(unsigned int)width withHeight:(unsigned int)height; +-(BOOL)startSession:(AVCaptureDevice *)device withWidth:(unsigned int)width withHeight:(unsigned int)height; -(CIImage *)snapshot; -(void)stopSession; diff --git a/macos/camera.m b/macos/camera.m index 41c114c..22669ee 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -26,10 +26,10 @@ @interface ImageSnap() -- (void)captureOutput:(QTCaptureOutput *)captureOutput +- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputVideoFrame:(CVImageBufferRef)videoFrame - withSampleBuffer:(QTSampleBuffer *)sampleBuffer - fromConnection:(QTCaptureConnection *)connection; + withSampleBuffer:(AVSampleBuffer *)sampleBuffer + fromConnection:(AVCaptureConnection *)connection; @end @@ -58,28 +58,28 @@ - (void)dealloc{ // Returns an array of video devices attached to this computer. + (NSArray *)videoDevices{ NSMutableArray *results = [NSMutableArray arrayWithCapacity:3]; - [results addObjectsFromArray:[QTCaptureDevice inputDevicesWithMediaType:QTMediaTypeVideo]]; - [results addObjectsFromArray:[QTCaptureDevice inputDevicesWithMediaType:QTMediaTypeMuxed]]; + [results addObjectsFromArray:[AVCaptureDevice inputDevicesWithMediaType:AVMediaTypeVideo]]; + [results addObjectsFromArray:[AVCaptureDevice inputDevicesWithMediaType:AVMediaTypeMuxed]]; return results; } // Returns the default video device or nil if none found. -+ (QTCaptureDevice *)defaultVideoDevice{ - QTCaptureDevice *device = nil; ++ (AVCaptureDevice *)defaultVideoDevice{ + AVCaptureDevice *device = nil; - device = [QTCaptureDevice defaultInputDeviceWithMediaType:QTMediaTypeVideo]; + device = [AVCaptureDevice defaultInputDeviceWithMediaType:AVMediaTypeVideo]; if( device == nil ){ - device = [QTCaptureDevice defaultInputDeviceWithMediaType:QTMediaTypeMuxed]; + device = [AVCaptureDevice defaultInputDeviceWithMediaType:AVMediaTypeMuxed]; } return device; } // Returns the named capture device or nil if not found. -+(QTCaptureDevice *)deviceNamed:(NSString *)name{ - QTCaptureDevice *result = nil; ++(AVCaptureDevice *)deviceNamed:(NSString *)name{ + AVCaptureDevice *result = nil; NSArray *devices = [ImageSnap videoDevices]; - for( QTCaptureDevice *device in devices ){ + for( AVCaptureDevice *device in devices ){ if ( [name isEqualToString:[device description]] ){ result = device; } // end if: match @@ -229,7 +229,7 @@ -(void)stopSession{ /** * Begins the capture session. Frames begin coming in. */ --(BOOL)startSession:(QTCaptureDevice *)device +-(BOOL)startSession:(AVCaptureDevice *)device withWidth:(unsigned int)width withHeight:(unsigned int)height { @@ -257,8 +257,8 @@ -(BOOL)startSession:(QTCaptureDevice *)device // Create the capture session - verbose( "\tCreating QTCaptureSession..." ); - mCaptureSession = [[QTCaptureSession alloc] init]; + verbose( "\tCreating AVCaptureSession..." ); + mCaptureSession = [[AVCaptureSession alloc] init]; verbose( "Done.\n"); if( ![device open:&error] ){ error( "\tCould not create capture session.\n" ); @@ -269,8 +269,8 @@ -(BOOL)startSession:(QTCaptureDevice *)device // Create input object from the device - verbose( "\tCreating QTCaptureDeviceInput with %s...", [[device description] UTF8String] ); - mCaptureDeviceInput = [[QTCaptureDeviceInput alloc] initWithDevice:device]; + verbose( "\tCreating AVCaptureDeviceInput with %s...", [[device description] UTF8String] ); + mCaptureDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:device]; verbose( "Done.\n"); if (![mCaptureSession addInput:mCaptureDeviceInput error:&error]) { error( "\tCould not convert device to input device.\n"); @@ -282,8 +282,8 @@ -(BOOL)startSession:(QTCaptureDevice *)device } // Decompressed video output - verbose( "\tCreating QTCaptureDecompressedVideoOutput..."); - mCaptureDecompressedVideoOutput = [[QTCaptureDecompressedVideoOutput alloc] init]; + verbose( "\tCreating AVCaptureVideoDataOutput..."); + mCaptureDecompressedVideoOutput = [[AVCaptureVideoDataOutput alloc] init]; [mCaptureDecompressedVideoOutput setDelegate:self]; [mCaptureDecompressedVideoOutput setPixelBufferAttributes:[NSDictionary dictionaryWithObjectsAndKeys: @@ -322,11 +322,11 @@ -(BOOL)startSession:(QTCaptureDevice *)device -// This delegate method is called whenever the QTCaptureDecompressedVideoOutput receives a frame -- (void)captureOutput:(QTCaptureOutput *)captureOutput +// This delegate method is called whenever the AVCaptureVideoDataOutput receives a frame +- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputVideoFrame:(CVImageBufferRef)videoFrame - withSampleBuffer:(QTSampleBuffer *)sampleBuffer - fromConnection:(QTCaptureConnection *)connection + withSampleBuffer:(AVSampleBuffer *)sampleBuffer + fromConnection:(AVCaptureConnection *)connection { verbose( "." ); if (videoFrame == nil ) { @@ -343,7 +343,6 @@ - (void)captureOutput:(QTCaptureOutput *)captureOutput mCurrentImageBuffer = videoFrame; } // end sync CVBufferRelease(imageBufferToRelease); - } @end @@ -354,7 +353,7 @@ - (void)captureOutput:(QTCaptureOutput *)captureOutput // static vars static int nbcams = 0; static ImageSnap **snap = NULL; -static QTCaptureDevice **device = NULL; +static AVCaptureDevice **device = NULL; static NSAutoreleasePool * pool = NULL; // start up all cameras found @@ -379,7 +378,7 @@ int initCameras(lua_State *L) { int k = 0; if ([deviceName count] > 0) { printf("found %ld video device(s):\n", [deviceName count]); - for( QTCaptureDevice *name in deviceName ){ + for( AVCaptureDevice *name in deviceName ){ printf( "%d: %s\n", k++, [[name description] UTF8String] ); } } else { @@ -393,9 +392,9 @@ int initCameras(lua_State *L) { nbcams = [deviceName count]; printf("only using the first %d camera(s)\n", nbcams); } - device = malloc(sizeof(QTCaptureDevice *)*nbcams); + device = malloc(sizeof(AVCaptureDevice *)*nbcams); int i = 0, j = 0; - for( QTCaptureDevice *dev in deviceName ) { + for( AVCaptureDevice *dev in deviceName ) { // next cam: for (int k=1; k<=nbcams; k++) { lua_rawgeti(L, 1, k); From 6153792efc2eca46e9a71321a487b18a2d7e2331 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 16:39:14 -0700 Subject: [PATCH 02/19] Comment out 11 sections causing errors/warnings --- macos/camera.m | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index 22669ee..ec5f610 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -25,12 +25,12 @@ #include @interface ImageSnap() - +/* - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputVideoFrame:(CVImageBufferRef)videoFrame withSampleBuffer:(AVSampleBuffer *)sampleBuffer fromConnection:(AVCaptureConnection *)connection; - +*/ @end @implementation ImageSnap @@ -58,8 +58,8 @@ - (void)dealloc{ // Returns an array of video devices attached to this computer. + (NSArray *)videoDevices{ NSMutableArray *results = [NSMutableArray arrayWithCapacity:3]; - [results addObjectsFromArray:[AVCaptureDevice inputDevicesWithMediaType:AVMediaTypeVideo]]; - [results addObjectsFromArray:[AVCaptureDevice inputDevicesWithMediaType:AVMediaTypeMuxed]]; +// [results addObjectsFromArray:[AVCaptureDevice inputDevicesWithMediaType:AVMediaTypeVideo]]; +// [results addObjectsFromArray:[AVCaptureDevice inputDevicesWithMediaType:AVMediaTypeMuxed]]; return results; } @@ -67,9 +67,9 @@ + (NSArray *)videoDevices{ + (AVCaptureDevice *)defaultVideoDevice{ AVCaptureDevice *device = nil; - device = [AVCaptureDevice defaultInputDeviceWithMediaType:AVMediaTypeVideo]; + //device = [AVCaptureDevice defaultInputDeviceWithMediaType:AVMediaTypeVideo]; if( device == nil ){ - device = [AVCaptureDevice defaultInputDeviceWithMediaType:AVMediaTypeMuxed]; + //device = [AVCaptureDevice defaultInputDeviceWithMediaType:AVMediaTypeMuxed]; } return device; } @@ -260,18 +260,21 @@ -(BOOL)startSession:(AVCaptureDevice *)device verbose( "\tCreating AVCaptureSession..." ); mCaptureSession = [[AVCaptureSession alloc] init]; verbose( "Done.\n"); + /* if( ![device open:&error] ){ error( "\tCould not create capture session.\n" ); [mCaptureSession release]; mCaptureSession = nil; return NO; } + */ // Create input object from the device verbose( "\tCreating AVCaptureDeviceInput with %s...", [[device description] UTF8String] ); - mCaptureDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:device]; + //mCaptureDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:device]; verbose( "Done.\n"); + /* if (![mCaptureSession addInput:mCaptureDeviceInput error:&error]) { error( "\tCould not convert device to input device.\n"); [mCaptureSession release]; @@ -280,19 +283,23 @@ -(BOOL)startSession:(AVCaptureDevice *)device mCaptureDeviceInput = nil; return NO; } + */ // Decompressed video output verbose( "\tCreating AVCaptureVideoDataOutput..."); mCaptureDecompressedVideoOutput = [[AVCaptureVideoDataOutput alloc] init]; - [mCaptureDecompressedVideoOutput setDelegate:self]; + //[mCaptureDecompressedVideoOutput setDelegate:self]; + /* [mCaptureDecompressedVideoOutput setPixelBufferAttributes:[NSDictionary dictionaryWithObjectsAndKeys: [NSNumber numberWithUnsignedInt:width], (id)kCVPixelBufferWidthKey, [NSNumber numberWithUnsignedInt:height], (id)kCVPixelBufferHeightKey, [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32ARGB], (id)kCVPixelBufferPixelFormatTypeKey, nil]]; + */ verbose( "Done.\n" ); + /* if (![mCaptureSession addOutput:mCaptureDecompressedVideoOutput error:&error]) { error( "\tCould not create decompressed output.\n"); [mCaptureSession release]; @@ -303,6 +310,7 @@ -(BOOL)startSession:(AVCaptureDevice *)device mCaptureDecompressedVideoOutput = nil; return NO; } + */ // Clear old image? verbose("\tEntering synchronized block to clear memory..."); @@ -323,6 +331,7 @@ -(BOOL)startSession:(AVCaptureDevice *)device // This delegate method is called whenever the AVCaptureVideoDataOutput receives a frame +/* - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputVideoFrame:(CVImageBufferRef)videoFrame withSampleBuffer:(AVSampleBuffer *)sampleBuffer @@ -344,6 +353,7 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput } // end sync CVBufferRelease(imageBufferToRelease); } +*/ @end From e873fcb8be9e844632f015dad4adf501de45f0ad Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 17:53:57 -0700 Subject: [PATCH 03/19] Fix (NSArray *)videoDevices --- macos/camera.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index ec5f610..2d29612 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -58,8 +58,8 @@ - (void)dealloc{ // Returns an array of video devices attached to this computer. + (NSArray *)videoDevices{ NSMutableArray *results = [NSMutableArray arrayWithCapacity:3]; -// [results addObjectsFromArray:[AVCaptureDevice inputDevicesWithMediaType:AVMediaTypeVideo]]; -// [results addObjectsFromArray:[AVCaptureDevice inputDevicesWithMediaType:AVMediaTypeMuxed]]; + [results addObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]]; + [results addObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]]; return results; } From cdf6b7b1eacfcfaa5e32d0b1de83ebd63d113778 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 17:54:51 -0700 Subject: [PATCH 04/19] Fix (AVCaptureDevice *)defaultVideoDevice --- macos/camera.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index 2d29612..858ccb4 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -67,9 +67,9 @@ + (NSArray *)videoDevices{ + (AVCaptureDevice *)defaultVideoDevice{ AVCaptureDevice *device = nil; - //device = [AVCaptureDevice defaultInputDeviceWithMediaType:AVMediaTypeVideo]; + device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; if( device == nil ){ - //device = [AVCaptureDevice defaultInputDeviceWithMediaType:AVMediaTypeMuxed]; + device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeMuxed]; } return device; } From 9270d83b604ce8b54a68927dc0b7e9b06cee80d3 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 18:04:12 -0700 Subject: [PATCH 05/19] Fix initWithDevice because it takes another arg now --- macos/camera.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macos/camera.m b/macos/camera.m index 858ccb4..1741fc0 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -272,7 +272,7 @@ -(BOOL)startSession:(AVCaptureDevice *)device // Create input object from the device verbose( "\tCreating AVCaptureDeviceInput with %s...", [[device description] UTF8String] ); - //mCaptureDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:device]; + mCaptureDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error]; verbose( "Done.\n"); /* if (![mCaptureSession addInput:mCaptureDeviceInput error:&error]) { From d85042092ba90ea618222ae048fdd0e14bb05ce0 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 18:26:21 -0700 Subject: [PATCH 06/19] Move error capture from addInput to initWithDevice --- macos/camera.m | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index 1741fc0..f26db79 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -273,9 +273,7 @@ -(BOOL)startSession:(AVCaptureDevice *)device // Create input object from the device verbose( "\tCreating AVCaptureDeviceInput with %s...", [[device description] UTF8String] ); mCaptureDeviceInput = [[AVCaptureDeviceInput alloc] initWithDevice:device error:&error]; - verbose( "Done.\n"); - /* - if (![mCaptureSession addInput:mCaptureDeviceInput error:&error]) { + if (!mCaptureDeviceInput) { error( "\tCould not convert device to input device.\n"); [mCaptureSession release]; [mCaptureDeviceInput release]; @@ -283,7 +281,8 @@ -(BOOL)startSession:(AVCaptureDevice *)device mCaptureDeviceInput = nil; return NO; } - */ + [mCaptureSession addInput:mCaptureDeviceInput]; + verbose( "Done.\n"); // Decompressed video output verbose( "\tCreating AVCaptureVideoDataOutput..."); From 1d9bd8ad78e9231054f238e35a2b81f7ef3ada21 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 18:30:23 -0700 Subject: [PATCH 07/19] The open() error check is handled creating AVCaptureDeviceInput --- macos/camera.m | 9 --------- 1 file changed, 9 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index f26db79..0890c56 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -260,15 +260,6 @@ -(BOOL)startSession:(AVCaptureDevice *)device verbose( "\tCreating AVCaptureSession..." ); mCaptureSession = [[AVCaptureSession alloc] init]; verbose( "Done.\n"); - /* - if( ![device open:&error] ){ - error( "\tCould not create capture session.\n" ); - [mCaptureSession release]; - mCaptureSession = nil; - return NO; - } - */ - // Create input object from the device verbose( "\tCreating AVCaptureDeviceInput with %s...", [[device description] UTF8String] ); From b05c87a250a61bc93790b3e3172d7a927aca01fd Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 18:39:09 -0700 Subject: [PATCH 08/19] Use a dispatch queue and specify --- macos/camera.h | 2 +- macos/camera.m | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/macos/camera.h b/macos/camera.h index d53c12a..ac7b4dd 100644 --- a/macos/camera.h +++ b/macos/camera.h @@ -12,7 +12,7 @@ BOOL g_verbose = NO; BOOL g_quiet = NO; -@interface ImageSnap : NSObject { +@interface ImageSnap : NSObject { AVCaptureSession *mCaptureSession; AVCaptureDeviceInput *mCaptureDeviceInput; diff --git a/macos/camera.m b/macos/camera.m index 0890c56..6798690 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -278,7 +278,9 @@ -(BOOL)startSession:(AVCaptureDevice *)device // Decompressed video output verbose( "\tCreating AVCaptureVideoDataOutput..."); mCaptureDecompressedVideoOutput = [[AVCaptureVideoDataOutput alloc] init]; - //[mCaptureDecompressedVideoOutput setDelegate:self]; + dispatch_queue_t queue = dispatch_queue_create("myQueue", NULL); + [mCaptureDecompressedVideoOutput setSampleBufferDelegate:self queue:queue]; + dispatch_release(queue); /* [mCaptureDecompressedVideoOutput setPixelBufferAttributes:[NSDictionary dictionaryWithObjectsAndKeys: From 65969c2da48dce10869bf37c556f9574247a7e94 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 18:55:33 -0700 Subject: [PATCH 09/19] Add two delegates for captureOutput and use CMSampleBufferGetImageBuffer --- macos/camera.m | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index 6798690..1cf7e81 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -25,12 +25,14 @@ #include @interface ImageSnap() -/* - (void)captureOutput:(AVCaptureOutput *)captureOutput - didOutputVideoFrame:(CVImageBufferRef)videoFrame - withSampleBuffer:(AVSampleBuffer *)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; -*/ + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection; + +- (void)captureOutput:(AVCaptureOutput *)captureOutput + didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection; + @end @implementation ImageSnap @@ -323,19 +325,14 @@ -(BOOL)startSession:(AVCaptureDevice *)device // This delegate method is called whenever the AVCaptureVideoDataOutput receives a frame -/* - (void)captureOutput:(AVCaptureOutput *)captureOutput - didOutputVideoFrame:(CVImageBufferRef)videoFrame - withSampleBuffer:(AVSampleBuffer *)sampleBuffer - fromConnection:(AVCaptureConnection *)connection + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection; { verbose( "." ); - if (videoFrame == nil ) { - verbose( "'nil' Frame captured.\n" ); - return; - } // Swap out old frame for new one + CVImageBufferRef videoFrame = CMSampleBufferGetImageBuffer(sampleBuffer); CVImageBufferRef imageBufferToRelease; CVBufferRetain(videoFrame); @@ -345,7 +342,14 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput } // end sync CVBufferRelease(imageBufferToRelease); } -*/ + +- (void)captureOutput:(AVCaptureOutput *)captureOutput + didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection; +{ + verbose( "." ); + verbose( "'nil' (dropped) Frame captured.\n" ); +} @end From 4249f0d53d37437a2bac69c39094cd2b69ec5459 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 19:02:01 -0700 Subject: [PATCH 10/19] Only kCVPixelBufferPixelFormatTypeKey is supported --- macos/camera.m | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index 1cf7e81..dcf91f8 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -284,13 +284,9 @@ -(BOOL)startSession:(AVCaptureDevice *)device [mCaptureDecompressedVideoOutput setSampleBufferDelegate:self queue:queue]; dispatch_release(queue); - /* - [mCaptureDecompressedVideoOutput setPixelBufferAttributes:[NSDictionary dictionaryWithObjectsAndKeys: - [NSNumber numberWithUnsignedInt:width], (id)kCVPixelBufferWidthKey, - [NSNumber numberWithUnsignedInt:height], (id)kCVPixelBufferHeightKey, - [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32ARGB], (id)kCVPixelBufferPixelFormatTypeKey, - nil]]; - */ + NSDictionary *newSettings = + @{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32ARGB) }; + mCaptureDecompressedVideoOutput.videoSettings = newSettings; verbose( "Done.\n" ); /* From 0b2920d39983e0cf8b89c8c0eb5b93a53131fc94 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 19:05:35 -0700 Subject: [PATCH 11/19] Looks like not much error checking for AVCaptureVideoDataOutput --- macos/camera.m | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index dcf91f8..a3c0d01 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -280,17 +280,8 @@ -(BOOL)startSession:(AVCaptureDevice *)device // Decompressed video output verbose( "\tCreating AVCaptureVideoDataOutput..."); mCaptureDecompressedVideoOutput = [[AVCaptureVideoDataOutput alloc] init]; - dispatch_queue_t queue = dispatch_queue_create("myQueue", NULL); - [mCaptureDecompressedVideoOutput setSampleBufferDelegate:self queue:queue]; - dispatch_release(queue); - - NSDictionary *newSettings = - @{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32ARGB) }; - mCaptureDecompressedVideoOutput.videoSettings = newSettings; - verbose( "Done.\n" ); - /* - if (![mCaptureSession addOutput:mCaptureDecompressedVideoOutput error:&error]) { + if (!mCaptureDecompressedVideoOutput) { error( "\tCould not create decompressed output.\n"); [mCaptureSession release]; [mCaptureDeviceInput release]; @@ -300,7 +291,14 @@ -(BOOL)startSession:(AVCaptureDevice *)device mCaptureDecompressedVideoOutput = nil; return NO; } - */ + dispatch_queue_t queue = dispatch_queue_create("myQueue", NULL); + [mCaptureDecompressedVideoOutput setSampleBufferDelegate:self queue:queue]; + dispatch_release(queue); + verbose( "Done.\n" ); + + NSDictionary *newSettings = + @{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32ARGB) }; + mCaptureDecompressedVideoOutput.videoSettings = newSettings; // Clear old image? verbose("\tEntering synchronized block to clear memory..."); From fc65064e1100f6871d19441e02c56c2e9919206b Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 19:06:34 -0700 Subject: [PATCH 12/19] Less strange indentation --- macos/camera.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index a3c0d01..af31b9b 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -27,11 +27,11 @@ @interface ImageSnap() - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; + fromConnection:(AVCaptureConnection *)connection; - (void)captureOutput:(AVCaptureOutput *)captureOutput didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; + fromConnection:(AVCaptureConnection *)connection; @end @@ -321,7 +321,7 @@ -(BOOL)startSession:(AVCaptureDevice *)device // This delegate method is called whenever the AVCaptureVideoDataOutput receives a frame - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; + fromConnection:(AVCaptureConnection *)connection; { verbose( "." ); @@ -339,7 +339,7 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput - (void)captureOutput:(AVCaptureOutput *)captureOutput didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; + fromConnection:(AVCaptureConnection *)connection; { verbose( "." ); verbose( "'nil' (dropped) Frame captured.\n" ); From 92bed8f2efdea806cab9b9944ef8323d5500ddfc Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 20:14:07 -0700 Subject: [PATCH 13/19] Try to match the old spacing style --- macos/camera.h | 2 +- macos/camera.m | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/macos/camera.h b/macos/camera.h index ac7b4dd..cf5d0ae 100644 --- a/macos/camera.h +++ b/macos/camera.h @@ -16,7 +16,7 @@ BOOL g_quiet = NO; AVCaptureSession *mCaptureSession; AVCaptureDeviceInput *mCaptureDeviceInput; - AVCaptureVideoDataOutput *mCaptureDecompressedVideoOutput; + AVCaptureVideoDataOutput *mCaptureDecompressedVideoOutput; CVImageBufferRef mCurrentImageBuffer; } diff --git a/macos/camera.m b/macos/camera.m index af31b9b..63e2565 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -27,11 +27,11 @@ @interface ImageSnap() - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; + fromConnection:(AVCaptureConnection *)connection; - (void)captureOutput:(AVCaptureOutput *)captureOutput didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; + fromConnection:(AVCaptureConnection *)connection; @end @@ -296,8 +296,7 @@ -(BOOL)startSession:(AVCaptureDevice *)device dispatch_release(queue); verbose( "Done.\n" ); - NSDictionary *newSettings = - @{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32ARGB) }; + NSDictionary *newSettings = @{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32ARGB) }; mCaptureDecompressedVideoOutput.videoSettings = newSettings; // Clear old image? @@ -321,7 +320,7 @@ -(BOOL)startSession:(AVCaptureDevice *)device // This delegate method is called whenever the AVCaptureVideoDataOutput receives a frame - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; + fromConnection:(AVCaptureConnection *)connection; { verbose( "." ); From 745a48900582ab41793198ae6acf214a133b6130 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 29 May 2017 22:13:46 -0700 Subject: [PATCH 14/19] Add missing call to addOutput --- macos/camera.m | 2 ++ 1 file changed, 2 insertions(+) diff --git a/macos/camera.m b/macos/camera.m index 63e2565..7e61b6e 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -291,6 +291,8 @@ -(BOOL)startSession:(AVCaptureDevice *)device mCaptureDecompressedVideoOutput = nil; return NO; } + [mCaptureSession addOutput:mCaptureDecompressedVideoOutput]; + dispatch_queue_t queue = dispatch_queue_create("myQueue", NULL); [mCaptureDecompressedVideoOutput setSampleBufferDelegate:self queue:queue]; dispatch_release(queue); From 89d6d575b914c392c7e612539003b5abfd43a3c9 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 5 Jun 2017 22:36:01 -0700 Subject: [PATCH 15/19] Use camopencv or cammacos, but not both --- CMakeLists.txt | 16 +++++++++++----- init.lua => init.lua.in | 7 +++++-- 2 files changed, 16 insertions(+), 7 deletions(-) rename init.lua => init.lua.in (92%) diff --git a/CMakeLists.txt b/CMakeLists.txt index d385cd1..2ea9717 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,17 +14,23 @@ endif (UNIX AND NOT APPLE) if (APPLE) add_subdirectory (opencv) - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + if (OpenCV_FOUND) + set(useOpenCV "true") + else() + if(NOT ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")) + message (ERROR "clang is required to build libcammacos") + endif() + set(useOpenCV "false") add_subdirectory (macos) - endif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + endif() endif (APPLE) if (NOT UNIX) message (ERROR "This package only builds on Unix platforms") endif (NOT UNIX) -install_files(/lua/camera init.lua) - SET(src) -SET(luasrc init.lua) +SET(luasrc "${CMAKE_BINARY_DIR}/init.lua") +configure_file(init.lua.in "${luasrc}") +install_files(/lua/camera "${luasrc}") ADD_TORCH_PACKAGE(camera "${src}" "${luasrc}" "Image Processing") diff --git a/init.lua b/init.lua.in similarity index 92% rename from init.lua rename to init.lua.in index 02da20f..227a80a 100644 --- a/init.lua +++ b/init.lua.in @@ -8,6 +8,9 @@ require 'xlua' ---------------------------------- -- load camera driver based on OS ---------------------------------- + +useOpenCV = @useOpenCV@ + if useOpenCV then if not xlua.require 'camopencv' then xlua.error('failed to load camopencv wrapper: verify that camopencv is installed') @@ -17,8 +20,8 @@ elseif sys.OS == 'linux' then xlua.error('failed to load video4linux wrapper: verify that you have v4l2 libs') end elseif sys.OS == 'macos' then - if not xlua.require 'camopencv' then - xlua.error('failed to load camopencv wrapper: verify that camopencv is installed') + if not xlua.require 'cammacos' then + xlua.error('failed to load cammacos wrapper: verify that cammacos is installed') end else xlua.error('no camera driver available for your OS, sorry :-(') From 1f6fd1240121ff2a2079f663ecd02a1aa70b3cb1 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 5 Jun 2017 22:42:20 -0700 Subject: [PATCH 16/19] Send OpenCV_FOUND up to parent scope --- opencv/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opencv/CMakeLists.txt b/opencv/CMakeLists.txt index b75efae..61c0f71 100644 --- a/opencv/CMakeLists.txt +++ b/opencv/CMakeLists.txt @@ -8,3 +8,5 @@ IF(OpenCV_FOUND) ADD_TORCH_PACKAGE(camopencv "${src}" "${luasrc}" "Image Processing") TARGET_LINK_LIBRARIES(camopencv luaT TH ${OpenCV_LIBS}) ENDIF(OpenCV_FOUND) + +set(OpenCV_FOUND ${OpenCV_FOUND} PARENT_SCOPE) From 43968def69e1e54e11d9ad32dbabc7695ff6d5f8 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 5 Jun 2017 23:08:56 -0700 Subject: [PATCH 17/19] Use the single-camera code --- macos/camera.m | 35 ++++++++++++++++++++--------------- macos/init.lua | 40 +++++++++++++--------------------------- 2 files changed, 33 insertions(+), 42 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index 7e61b6e..878119b 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -431,16 +431,15 @@ int initCameras(lua_State *L) { return 1; } -// grab next frames -int grabFrames(lua_State *L) { +// grab next frame +int grabFrame(lua_State *L) { // grab pixels for each camera for (int i=0; istride[1]; + int m1 = tensor->stride[2]; + int m2 = tensor->stride[0]; + int i, j, k; + const int nChannels = 3; + for (i = 0; i < height; i++) { + for (j = 0, k = 0; j < width; j++, k+= m1) { + dst[k] = bytes[i*bytesPerRow + j*nChannels + 2]/255.; + dst[k+m2] = bytes[i*bytesPerRow + j*nChannels + 1]/255.; + dst[k+2*m2] = bytes[i*bytesPerRow + j*nChannels + 0]/255.; + } + dst += m0; } // cleanup @@ -497,7 +502,7 @@ int releaseCameras(lua_State *L) { // Register functions into lua space static const struct luaL_reg cammacos [] = { {"initCameras", initCameras}, - {"grabFrames", grabFrames}, + {"grabFrame", grabFrame}, {"releaseCameras", releaseCameras}, {NULL, NULL} /* sentinel */ }; diff --git a/macos/init.lua b/macos/init.lua index cc7cafb..1414baa 100644 --- a/macos/init.lua +++ b/macos/init.lua @@ -38,37 +38,23 @@ function Camera:__init(...) end -- buffers - self.tensorsized = {} - self.buffer = {} - self.tensortyped = {} - for i = 1,#self.idx do - self.tensorsized[i] = torch.FloatTensor(3, height, width) - self.buffer[i] = torch.FloatTensor() - self.tensortyped[i] = torch.Tensor(3, height, width) - end + self.tensorsized = torch.FloatTensor(3, height, width) + self.buffer = torch.FloatTensor() + self.tensortyped = torch.Tensor(3, height, width) end function Camera:forward() - -- grab all frames - libcammacos.grabFrames(self.buffer) - - -- process all frames - for i = 1,#self.idx do - -- resize frames - if self.tensorsized[i]:size(2) ~= self.buffer[i]:size(2) or self.tensorsized[i]:size(3) ~= self.buffer[i]:size(3) then - image.scale(self.tensorsized[i],self.buffer[i]) - else - self.tensorsized[i] = self.buffer[i] - end - -- retype frames - if self.tensortyped[i]:type() ~= self.tensorsized[i]:type() then - self.tensortyped[i]:copy(self.tensorsized[i]) - else - self.tensortyped[i] = self.tensorsized[i] - end + libcammacos.grabFrame(self.idx, self.buffer) + if self.tensorsized:size(2) ~= self.buffer:size(2) or self.tensorsized:size(3) ~= self.buffer:size(3) then + image.scale(self.tensorsized, self.buffer) + else + self.tensorsized = self.buffer + end + if self.tensortyped:type() ~= self.tensorsized:type() then + self.tensortyped:copy(self.tensorsized) + else + self.tensortyped = self.tensorsized end - - -- done return self.tensortyped end From d95b75ed77a5d2c2fd4d40f986886812e6136446 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 5 Jun 2017 23:14:11 -0700 Subject: [PATCH 18/19] Fix multi-camera interface to grab from index idx --- macos/camera.m | 84 ++++++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 44 deletions(-) diff --git a/macos/camera.m b/macos/camera.m index 878119b..a388fbb 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -434,52 +434,48 @@ int initCameras(lua_State *L) { // grab next frame int grabFrame(lua_State *L) { - // grab pixels for each camera - for (int i=0; istride[1]; - int m1 = tensor->stride[2]; - int m2 = tensor->stride[0]; - int i, j, k; - const int nChannels = 3; - for (i = 0; i < height; i++) { - for (j = 0, k = 0; j < width; j++, k+= m1) { - dst[k] = bytes[i*bytesPerRow + j*nChannels + 2]/255.; - dst[k+m2] = bytes[i*bytesPerRow + j*nChannels + 1]/255.; - dst[k+2*m2] = bytes[i*bytesPerRow + j*nChannels + 0]/255.; - } - dst += m0; - } - - // cleanup - [imageRep release]; - [image release]; + // get next tensor + const int idx = lua_tonumber(L, 1); + THFloatTensor * tensor = luaT_toudata(L, 2, "torch.FloatTensor"); + + // grab frame + verbose("grabbing image %d\n", idx); + CIImage *image = [snap[idx] snapshot]; + verbose("grabbed\n"); + + // export to bitmap + NSBitmapImageRep *imageRep = [[NSBitmapImageRep alloc] initWithCIImage:image]; + + // get info + NSSize size = [imageRep size]; + int bytesPerRow = [imageRep bytesPerRow]; + unsigned char *bytes = [imageRep bitmapData]; + + // resize dest + long width = size.width; + long height = size.height; + THFloatTensor_resize3d(tensor, 3, height, width); + + // copy pixels to tensor + float *dst = THFloatTensor_data(tensor); + int m0 = tensor->stride[1]; + int m1 = tensor->stride[2]; + int m2 = tensor->stride[0]; + int i, j, k; + const int nChannels = 3; + for (i = 0; i < height; i++) { + for (j = 0, k = 0; j < width; j++, k+= m1) { + dst[k] = bytes[i*bytesPerRow + j*nChannels + 2]/255.; + dst[k+m2] = bytes[i*bytesPerRow + j*nChannels + 1]/255.; + dst[k+2*m2] = bytes[i*bytesPerRow + j*nChannels + 0]/255.; + } + dst += m0; } + // cleanup + [imageRep release]; + [image release]; + // done return 0; } From 3d1b6d4948a09874f86c11c03ed5a8c15c28cb07 Mon Sep 17 00:00:00 2001 From: James Donald Date: Mon, 5 Jun 2017 23:25:06 -0700 Subject: [PATCH 19/19] Don't hardcode nChannels, fixes colorization --- macos/camera.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macos/camera.m b/macos/camera.m index a388fbb..983e169 100644 --- a/macos/camera.m +++ b/macos/camera.m @@ -462,7 +462,7 @@ int grabFrame(lua_State *L) { int m1 = tensor->stride[2]; int m2 = tensor->stride[0]; int i, j, k; - const int nChannels = 3; + const int nChannels = [imageRep samplesPerPixel]; for (i = 0; i < height; i++) { for (j = 0, k = 0; j < width; j++, k+= m1) { dst[k] = bytes[i*bytesPerRow + j*nChannels + 2]/255.;