/**************************************************************************/ /* camera_macos.mm */ /**************************************************************************/ /* This file is part of: */ /* GODOT ENGINE */ /* https://godotengine.org */ /**************************************************************************/ /* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ /* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ /* "Software"), to deal in the Software without restriction, including */ /* without limitation the rights to use, copy, modify, merge, publish, */ /* distribute, sublicense, and/or sell copies of the Software, and to */ /* permit persons to whom the Software is furnished to do so, subject to */ /* the following conditions: */ /* */ /* The above copyright notice and this permission notice shall be */ /* included in all copies or substantial portions of the Software. */ /* */ /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ /* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ /* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ /* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ /* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ /* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /**************************************************************************/ ///@TODO this is a near duplicate of CameraIOS, we should find a way to combine those to minimize code duplication!!!! // If you fix something here, make sure you fix it there as well! #import "camera_macos.h" #include "servers/camera/camera_feed.h" #import ////////////////////////////////////////////////////////////////////////// // MyCaptureSession - This is a little helper class so we can capture our frames @interface MyCaptureSession : AVCaptureSession { Ref feed; size_t width[2]; size_t height[2]; Vector img_data[2]; AVCaptureDeviceInput *input; AVCaptureVideoDataOutput *output; } @end @implementation MyCaptureSession - (id)initForFeed:(Ref)p_feed andDevice:(AVCaptureDevice *)p_device { if (self = [super init]) { NSError *error; feed = p_feed; width[0] = 0; height[0] = 0; width[1] = 0; height[1] = 0; #ifdef APPLE_EMBEDDED_ENABLED [p_device lockForConfiguration:&error]; [p_device setFocusMode:AVCaptureFocusModeLocked]; [p_device setExposureMode:AVCaptureExposureModeLocked]; [p_device setWhiteBalanceMode:AVCaptureWhiteBalanceModeLocked]; [p_device unlockForConfiguration]; #endif // APPLE_EMBEDDED_ENABLED [self beginConfiguration]; #ifdef APPLE_EMBEDDED_ENABLED self.sessionPreset = AVCaptureSessionPreset1280x720; #endif // APPLE_EMBEDDED_ENABLED input = [AVCaptureDeviceInput deviceInputWithDevice:p_device error:&error]; if (!input) { print_line("Couldn't get input device for camera"); [self commitConfiguration]; return nil; } [self addInput:input]; output = [AVCaptureVideoDataOutput new]; if (!output) { print_line("Couldn't get output device for camera"); [self commitConfiguration]; return nil; } NSDictionary *settings = @{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) }; output.videoSettings = settings; // discard if the data output queue is blocked (as we process the still image) [output setAlwaysDiscardsLateVideoFrames:YES]; // now set ourselves as the delegate to receive new frames. [output setSampleBufferDelegate:self queue:dispatch_get_main_queue()]; // this takes ownership [self addOutput:output]; [self commitConfiguration]; // kick off our session.. [self startRunning]; }; return self; } - (void)cleanup { // stop running [self stopRunning]; // cleanup [self beginConfiguration]; // remove input if (input) { [self removeInput:input]; // don't release this input = nullptr; } // free up our output if (output) { [self removeOutput:output]; [output setSampleBufferDelegate:nil queue:nullptr]; output = nullptr; } [self commitConfiguration]; } - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { // This gets called every time our camera has a new image for us to process. // May need to investigate in a way to throttle this if we get more images then we're rendering frames.. // For now, version 1, we're just doing the bare minimum to make this work... CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); if (pixelBuffer == nullptr) { return; } // It says that we need to lock this on the documentation pages but it's not in the samples // need to lock our base address so we can access our pixel buffers, better safe then sorry? CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); // Check if we have the expected number of planes (Y and CbCr). size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer); if (planeCount < 2) { static bool plane_count_error_logged = false; if (!plane_count_error_logged) { ERR_PRINT("Unexpected plane count in pixel buffer (expected 2, got " + itos(planeCount) + ")"); plane_count_error_logged = true; } CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); return; } // get our buffers unsigned char *dataY = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0); unsigned char *dataCbCr = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1); if (dataY == nullptr || dataCbCr == nullptr) { static bool buffer_access_error_logged = false; if (!buffer_access_error_logged) { ERR_PRINT("Couldn't access pixel buffer plane data"); buffer_access_error_logged = true; } CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); return; } Ref img[2]; { // do Y size_t new_width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0); size_t new_height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0); if ((width[0] != new_width) || (height[0] != new_height)) { width[0] = new_width; height[0] = new_height; img_data[0].resize(new_width * new_height); } uint8_t *w = img_data[0].ptrw(); memcpy(w, dataY, new_width * new_height); img[0].instantiate(); img[0]->set_data(new_width, new_height, 0, Image::FORMAT_R8, img_data[0]); } { // do CbCr size_t new_width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1); size_t new_height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1); if ((width[1] != new_width) || (height[1] != new_height)) { width[1] = new_width; height[1] = new_height; img_data[1].resize(2 * new_width * new_height); } uint8_t *w = img_data[1].ptrw(); memcpy(w, dataCbCr, 2 * new_width * new_height); ///TODO OpenGL doesn't support FORMAT_RG8, need to do some form of conversion img[1].instantiate(); img[1]->set_data(new_width, new_height, 0, Image::FORMAT_RG8, img_data[1]); } // set our texture... feed->set_ycbcr_images(img[0], img[1]); // and unlock CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); } @end ////////////////////////////////////////////////////////////////////////// // CameraFeedMacOS - Subclass for camera feeds in macOS class CameraFeedMacOS : public CameraFeed { GDSOFTCLASS(CameraFeedMacOS, CameraFeed); private: AVCaptureDevice *device; MyCaptureSession *capture_session; public: AVCaptureDevice *get_device() const; CameraFeedMacOS(); ~CameraFeedMacOS(); void set_device(AVCaptureDevice *p_device); bool activate_feed() override; void deactivate_feed() override; }; AVCaptureDevice *CameraFeedMacOS::get_device() const { return device; } CameraFeedMacOS::CameraFeedMacOS() { device = nullptr; capture_session = nullptr; } CameraFeedMacOS::~CameraFeedMacOS() { if (is_active()) { deactivate_feed(); } } void CameraFeedMacOS::set_device(AVCaptureDevice *p_device) { device = p_device; // get some info NSString *device_name = p_device.localizedName; name = String::utf8(device_name.UTF8String); position = CameraFeed::FEED_UNSPECIFIED; if ([p_device position] == AVCaptureDevicePositionBack) { position = CameraFeed::FEED_BACK; } else if ([p_device position] == AVCaptureDevicePositionFront) { position = CameraFeed::FEED_FRONT; }; } bool CameraFeedMacOS::activate_feed() { if (capture_session) { // Already recording. return true; } // Start camera capture, check permission. if (@available(macOS 10.14, *)) { AVAuthorizationStatus status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]; if (status == AVAuthorizationStatusAuthorized) { capture_session = [[MyCaptureSession alloc] initForFeed:this andDevice:device]; return capture_session != nullptr; } else if (status == AVAuthorizationStatusNotDetermined) { // Request permission asynchronously. [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo completionHandler:^(BOOL granted) { if (granted) { capture_session = [[MyCaptureSession alloc] initForFeed:this andDevice:device]; } }]; return false; } else if (status == AVAuthorizationStatusDenied) { print_line("Camera permission denied by user."); return false; } else if (status == AVAuthorizationStatusRestricted) { print_line("Camera access restricted."); return false; } return false; } else { capture_session = [[MyCaptureSession alloc] initForFeed:this andDevice:device]; return capture_session != nullptr; } } void CameraFeedMacOS::deactivate_feed() { // end camera capture if we have one if (capture_session) { [capture_session cleanup]; capture_session = nullptr; }; } ////////////////////////////////////////////////////////////////////////// // MyDeviceNotifications - This is a little helper class gets notifications // when devices are connected/disconnected @interface MyDeviceNotifications : NSObject { CameraMacOS *camera_server; } @end @implementation MyDeviceNotifications - (void)devices_changed:(NSNotification *)notification { camera_server->update_feeds(); } - (id)initForServer:(CameraMacOS *)p_server { if (self = [super init]) { camera_server = p_server; [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(devices_changed:) name:AVCaptureDeviceWasConnectedNotification object:nil]; [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(devices_changed:) name:AVCaptureDeviceWasDisconnectedNotification object:nil]; }; return self; } - (void)dealloc { // remove notifications [[NSNotificationCenter defaultCenter] removeObserver:self name:AVCaptureDeviceWasConnectedNotification object:nil]; [[NSNotificationCenter defaultCenter] removeObserver:self name:AVCaptureDeviceWasDisconnectedNotification object:nil]; } @end MyDeviceNotifications *device_notifications = nil; ////////////////////////////////////////////////////////////////////////// // CameraMacOS - Subclass for our camera server on macOS void CameraMacOS::update_feeds() { NSArray *devices = nullptr; #if defined(__x86_64__) if (@available(macOS 10.15, *)) { #endif AVCaptureDeviceDiscoverySession *session; if (@available(macOS 14.0, *)) { session = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:[NSArray arrayWithObjects:AVCaptureDeviceTypeExternal, AVCaptureDeviceTypeBuiltInWideAngleCamera, AVCaptureDeviceTypeContinuityCamera, nil] mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionUnspecified]; } else { session = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:[NSArray arrayWithObjects:AVCaptureDeviceTypeExternalUnknown, AVCaptureDeviceTypeBuiltInWideAngleCamera, nil] mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionUnspecified]; } devices = session.devices; #if defined(__x86_64__) } else { devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; } #endif // Deactivate feeds that are gone before removing them. for (int i = feeds.size() - 1; i >= 0; i--) { Ref feed = (Ref)feeds[i]; if (feed.is_null()) { continue; } if (![devices containsObject:feed->get_device()]) { if (feed->is_active()) { feed->deactivate_feed(); } remove_feed(feed); }; }; for (AVCaptureDevice *device in devices) { bool found = false; for (int i = 0; i < feeds.size() && !found; i++) { Ref feed = (Ref)feeds[i]; if (feed.is_null()) { continue; } if (feed->get_device() == device) { found = true; }; }; if (!found) { Ref newfeed; newfeed.instantiate(); newfeed->set_device(device); add_feed(newfeed); }; }; emit_signal(SNAME(CameraServer::feeds_updated_signal_name)); } void CameraMacOS::set_monitoring_feeds(bool p_monitoring_feeds) { if (p_monitoring_feeds == monitoring_feeds) { return; } CameraServer::set_monitoring_feeds(p_monitoring_feeds); if (p_monitoring_feeds) { // Find available cameras we have at this time. update_feeds(); // Get notified on feed changes. device_notifications = [[MyDeviceNotifications alloc] initForServer:this]; } else { // Stop monitoring feed changes. device_notifications = nil; } }