diff --git a/ARKit/arkitdemo.xcodeproj/project.pbxproj b/ARKit/arkitdemo.xcodeproj/project.pbxproj new file mode 100644 index 0000000..769acc8 --- /dev/null +++ b/ARKit/arkitdemo.xcodeproj/project.pbxproj @@ -0,0 +1,309 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 48; + objects = { + +/* Begin PBXBuildFile section */ + 3C335AC71F27D12500FF39A5 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C335AC61F27D12500FF39A5 /* AppDelegate.swift */; }; + 3C335AC91F27D12500FF39A5 /* Shaders.metal in Sources */ = {isa = PBXBuildFile; fileRef = 3C335AC81F27D12500FF39A5 /* Shaders.metal */; }; + 3C335ACC1F27D12500FF39A5 /* Renderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C335ACB1F27D12500FF39A5 /* Renderer.swift */; }; + 3C335ACE1F27D12500FF39A5 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C335ACD1F27D12500FF39A5 /* ViewController.swift */; }; + 3C335AD11F27D12500FF39A5 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 3C335ACF1F27D12500FF39A5 /* Main.storyboard */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 3C335AC31F27D12500FF39A5 /* arkitdemo.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = arkitdemo.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 3C335AC61F27D12500FF39A5 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; + 3C335AC81F27D12500FF39A5 /* Shaders.metal */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.metal; path = Shaders.metal; sourceTree = ""; }; + 3C335ACB1F27D12500FF39A5 /* Renderer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Renderer.swift; sourceTree = ""; }; + 3C335ACD1F27D12500FF39A5 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; + 3C335AD01F27D12500FF39A5 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; + 3C335AD71F27D12500FF39A5 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 3C335AC01F27D12500FF39A5 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 3C335ABA1F27D12500FF39A5 = { + isa = PBXGroup; + children = ( + 3C335AC51F27D12500FF39A5 /* arkitdemo */, + 3C335AC41F27D12500FF39A5 /* Products */, + ); + sourceTree = ""; + }; + 3C335AC41F27D12500FF39A5 /* Products */ = { + isa = PBXGroup; + children = ( + 3C335AC31F27D12500FF39A5 /* arkitdemo.app */, + ); + name = Products; + sourceTree = ""; + }; + 3C335AC51F27D12500FF39A5 /* arkitdemo */ = { + isa = PBXGroup; + children = ( + 3C335AC81F27D12500FF39A5 /* Shaders.metal */, + 3C335ACB1F27D12500FF39A5 /* Renderer.swift */, + 3C335ACD1F27D12500FF39A5 /* ViewController.swift */, + 3C335ACF1F27D12500FF39A5 /* Main.storyboard */, + 3C335AD71F27D12500FF39A5 /* Info.plist */, + 3C335AC61F27D12500FF39A5 /* AppDelegate.swift */, + ); + path = arkitdemo; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 3C335AC21F27D12500FF39A5 /* arkitdemo */ = { + isa = PBXNativeTarget; + buildConfigurationList = 3C335ADB1F27D12500FF39A5 /* Build configuration list for PBXNativeTarget "arkitdemo" */; + buildPhases = ( + 3C335ABF1F27D12500FF39A5 /* Sources */, + 3C335AC01F27D12500FF39A5 /* Frameworks */, + 3C335AC11F27D12500FF39A5 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = arkitdemo; + productName = arkitdemo; + productReference = 3C335AC31F27D12500FF39A5 /* arkitdemo.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 3C335ABB1F27D12500FF39A5 /* Project object */ = { + isa = PBXProject; + attributes = { + LastSwiftUpdateCheck = 0900; + LastUpgradeCheck = 0900; + ORGANIZATIONNAME = "Marius Horga"; + TargetAttributes = { + 3C335AC21F27D12500FF39A5 = { + CreatedOnToolsVersion = 9.0; + }; + }; + }; + buildConfigurationList = 3C335ABE1F27D12500FF39A5 /* Build configuration list for PBXProject "arkitdemo" */; + compatibilityVersion = "Xcode 8.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 3C335ABA1F27D12500FF39A5; + productRefGroup = 3C335AC41F27D12500FF39A5 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 3C335AC21F27D12500FF39A5 /* arkitdemo */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 3C335AC11F27D12500FF39A5 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 3C335AD11F27D12500FF39A5 /* Main.storyboard in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 3C335ABF1F27D12500FF39A5 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 3C335AC91F27D12500FF39A5 /* Shaders.metal in Sources */, + 3C335ACC1F27D12500FF39A5 /* Renderer.swift in Sources */, + 3C335ACE1F27D12500FF39A5 /* ViewController.swift in Sources */, + 3C335AC71F27D12500FF39A5 /* AppDelegate.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXVariantGroup section */ + 3C335ACF1F27D12500FF39A5 /* Main.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 3C335AD01F27D12500FF39A5 /* Base */, + ); + name = Main.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 3C335AD91F27D12500FF39A5 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_IDENTITY = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 11.0; + MTL_ENABLE_DEBUG_INFO = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + 3C335ADA1F27D12500FF39A5 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_IDENTITY = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 11.0; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 3C335ADC1F27D12500FF39A5 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + DEVELOPMENT_TEAM = C2X9Q6VFVJ; + INFOPLIST_FILE = arkitdemo/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = org.metalkit.arkitdemo; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 4.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 3C335ADD1F27D12500FF39A5 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + DEVELOPMENT_TEAM = C2X9Q6VFVJ; + INFOPLIST_FILE = arkitdemo/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = org.metalkit.arkitdemo; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 4.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 3C335ABE1F27D12500FF39A5 /* Build configuration list for PBXProject "arkitdemo" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 3C335AD91F27D12500FF39A5 /* Debug */, + 3C335ADA1F27D12500FF39A5 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 3C335ADB1F27D12500FF39A5 /* Build configuration list for PBXNativeTarget "arkitdemo" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 3C335ADC1F27D12500FF39A5 /* Debug */, + 3C335ADD1F27D12500FF39A5 /* Release */, + ); + defaultConfigurationIsVisible = 0; + }; +/* End XCConfigurationList section */ + }; + rootObject = 3C335ABB1F27D12500FF39A5 /* Project object */; +} diff --git a/ARKit/arkitdemo.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/ARKit/arkitdemo.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..0715532 --- /dev/null +++ b/ARKit/arkitdemo.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/ARKit/arkitdemo.xcodeproj/project.xcworkspace/xcuserdata/marius.xcuserdatad/UserInterfaceState.xcuserstate b/ARKit/arkitdemo.xcodeproj/project.xcworkspace/xcuserdata/marius.xcuserdatad/UserInterfaceState.xcuserstate new file mode 100644 index 0000000..42ae98c Binary files /dev/null and b/ARKit/arkitdemo.xcodeproj/project.xcworkspace/xcuserdata/marius.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/ARKit/arkitdemo.xcodeproj/project.xcworkspace/xcuserdata/mhorga.xcuserdatad/UserInterfaceState.xcuserstate b/ARKit/arkitdemo.xcodeproj/project.xcworkspace/xcuserdata/mhorga.xcuserdatad/UserInterfaceState.xcuserstate new file mode 100644 index 0000000..ce58460 Binary files /dev/null and b/ARKit/arkitdemo.xcodeproj/project.xcworkspace/xcuserdata/mhorga.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/ARKit/arkitdemo.xcodeproj/xcuserdata/marius.xcuserdatad/xcschemes/xcschememanagement.plist b/ARKit/arkitdemo.xcodeproj/xcuserdata/marius.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000..f35774a --- /dev/null +++ b/ARKit/arkitdemo.xcodeproj/xcuserdata/marius.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,14 @@ + + + + + SchemeUserState + + arkitdemo.xcscheme + + orderHint + 0 + + + + diff --git a/ARKit/arkitdemo.xcodeproj/xcuserdata/mhorga.xcuserdatad/xcschemes/xcschememanagement.plist b/ARKit/arkitdemo.xcodeproj/xcuserdata/mhorga.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000..f35774a --- /dev/null +++ b/ARKit/arkitdemo.xcodeproj/xcuserdata/mhorga.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,14 @@ + + + + + SchemeUserState + + arkitdemo.xcscheme + + orderHint + 0 + + + + diff --git a/ARKit/arkitdemo/AppDelegate.swift b/ARKit/arkitdemo/AppDelegate.swift new file mode 100644 index 0000000..5b60f07 --- /dev/null +++ b/ARKit/arkitdemo/AppDelegate.swift @@ -0,0 +1,7 @@ + +import UIKit + +@UIApplicationMain +class AppDelegate: UIResponder, UIApplicationDelegate { + var window: UIWindow? +} diff --git a/ARKit/arkitdemo/Base.lproj/Main.storyboard b/ARKit/arkitdemo/Base.lproj/Main.storyboard new file mode 100644 index 0000000..07ab49e --- /dev/null +++ b/ARKit/arkitdemo/Base.lproj/Main.storyboard @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ARKit/arkitdemo/Info.plist b/ARKit/arkitdemo/Info.plist new file mode 100644 index 0000000..674bae9 --- /dev/null +++ b/ARKit/arkitdemo/Info.plist @@ -0,0 +1,47 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + NSCameraUsageDescription + This application will use the camera for Augmented Reality. + UILaunchStoryboardName + Main + UIMainStoryboardFile + Main + UIRequiredDeviceCapabilities + + armv7 + metal + + UIRequiresFullScreen + + UIStatusBarHidden + + UISupportedInterfaceOrientations + + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationLandscapeRight + + + diff --git a/ARKit/arkitdemo/Renderer.swift b/ARKit/arkitdemo/Renderer.swift new file mode 100644 index 0000000..1c875c8 --- /dev/null +++ b/ARKit/arkitdemo/Renderer.swift @@ -0,0 +1,298 @@ + +import MetalKit +import ARKit + +protocol RenderDestinationProvider { + var currentRenderPassDescriptor: MTLRenderPassDescriptor? { get } + var currentDrawable: CAMetalDrawable? { get } + var colorPixelFormat: MTLPixelFormat { get set } + var depthStencilPixelFormat: MTLPixelFormat { get set } + var sampleCount: Int { get set } +} + +let maxBuffersInFlight: Int = 3 +let maxAnchorInstanceCount: Int = 64 + +class Renderer { + + let session: ARSession + let device: MTLDevice + var commandQueue: MTLCommandQueue! + let inFlightSemaphore = DispatchSemaphore(value: maxBuffersInFlight) + var renderDestination: RenderDestinationProvider + var sharedUniformBuffer: MTLBuffer! + var anchorUniformBuffer: MTLBuffer! + var imagePlaneVertexBuffer: MTLBuffer! + var capturedImagePipelineState: MTLRenderPipelineState! + var capturedImageDepthState: MTLDepthStencilState! + var anchorPipelineState: MTLRenderPipelineState! + var anchorDepthState: MTLDepthStencilState! + var capturedImageTextureY: MTLTexture! + var capturedImageTextureCbCr: MTLTexture! + var capturedImageTextureCache: CVMetalTextureCache! + var geometryVertexDescriptor: MTLVertexDescriptor! + var mesh: MTKMesh! + var uniformBufferIndex: Int = 0 + var sharedUniformBufferOffset: Int = 0 + var anchorUniformBufferOffset: Int = 0 + var sharedUniformBufferAddress: UnsafeMutableRawPointer! + var anchorUniformBufferAddress: UnsafeMutableRawPointer! + var anchorInstanceCount: Int = 0 + var viewportSize: CGSize = CGSize() + var viewportSizeDidChange: Bool = false + + struct SharedUniforms { + var projectionMatrix: matrix_float4x4 + var viewMatrix: matrix_float4x4 + var ambientLightColor: vector_float3 + var directionalLightDirection: vector_float3 + var directionalLightColor: vector_float3 + var materialShininess: Float + } + + struct InstanceUniforms { + var modelMatrix: matrix_float4x4 + } + + let alignedSharedUniformSize = (MemoryLayout.size & ~0xFF) + 0x100 + let alignedInstanceUniformSize = ((MemoryLayout.size * maxAnchorInstanceCount) & ~0xFF) + 0x100 + let planeVertexData: [Float] = [-1, -1, 0, 1, + 1, -1, 1, 1, + -1, 1, 0, 0, + 1, 1, 1, 0] + + init(session: ARSession, metalDevice device: MTLDevice, renderDestination: RenderDestinationProvider) { + self.session = session + self.device = device + self.renderDestination = renderDestination + setupPipeline() + setupAssets() + } + + func setupPipeline() { + renderDestination.depthStencilPixelFormat = .depth32Float_stencil8 + renderDestination.colorPixelFormat = .bgra8Unorm + renderDestination.sampleCount = 1 + let sharedUniformBufferSize = alignedSharedUniformSize * maxBuffersInFlight + let anchorUniformBufferSize = alignedInstanceUniformSize * maxBuffersInFlight + sharedUniformBuffer = device.makeBuffer(length: sharedUniformBufferSize, options: .storageModeShared) + anchorUniformBuffer = device.makeBuffer(length: anchorUniformBufferSize, options: .storageModeShared) + let imagePlaneVertexDataCount = planeVertexData.count * MemoryLayout.size + imagePlaneVertexBuffer = device.makeBuffer(bytes: planeVertexData, length: imagePlaneVertexDataCount, options: []) + let defaultLibrary = device.makeDefaultLibrary()! + let capturedImageVertexFunction = defaultLibrary.makeFunction(name: "capturedImageVertexTransform")! + let capturedImageFragmentFunction = defaultLibrary.makeFunction(name: "capturedImageFragmentShader")! + let imagePlaneVertexDescriptor = MTLVertexDescriptor() + imagePlaneVertexDescriptor.attributes[0].format = .float2 + imagePlaneVertexDescriptor.attributes[0].offset = 0 + imagePlaneVertexDescriptor.attributes[0].bufferIndex = 0 + imagePlaneVertexDescriptor.attributes[1].format = .float2 + imagePlaneVertexDescriptor.attributes[1].offset = 8 + imagePlaneVertexDescriptor.attributes[1].bufferIndex = 0 + imagePlaneVertexDescriptor.layouts[0].stride = 16 + imagePlaneVertexDescriptor.layouts[0].stepRate = 1 + imagePlaneVertexDescriptor.layouts[0].stepFunction = .perVertex + let capturedImagePipelineStateDescriptor = MTLRenderPipelineDescriptor() + capturedImagePipelineStateDescriptor.label = "MyCapturedImagePipeline" + capturedImagePipelineStateDescriptor.sampleCount = renderDestination.sampleCount + capturedImagePipelineStateDescriptor.vertexFunction = capturedImageVertexFunction + capturedImagePipelineStateDescriptor.fragmentFunction = capturedImageFragmentFunction + capturedImagePipelineStateDescriptor.vertexDescriptor = imagePlaneVertexDescriptor + capturedImagePipelineStateDescriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat + capturedImagePipelineStateDescriptor.depthAttachmentPixelFormat = renderDestination.depthStencilPixelFormat + capturedImagePipelineStateDescriptor.stencilAttachmentPixelFormat = renderDestination.depthStencilPixelFormat + do { try capturedImagePipelineState = device.makeRenderPipelineState(descriptor: capturedImagePipelineStateDescriptor) } + catch let error { print("Failed to created captured image pipeline state, error \(error)") } + let capturedImageDepthStateDescriptor = MTLDepthStencilDescriptor() + capturedImageDepthStateDescriptor.depthCompareFunction = .always + capturedImageDepthStateDescriptor.isDepthWriteEnabled = false + capturedImageDepthState = device.makeDepthStencilState(descriptor: capturedImageDepthStateDescriptor) + var textureCache: CVMetalTextureCache? + CVMetalTextureCacheCreate(nil, nil, device, nil, &textureCache) + capturedImageTextureCache = textureCache + let anchorGeometryVertexFunction = defaultLibrary.makeFunction(name: "anchorGeometryVertexTransform")! + let anchorGeometryFragmentFunction = defaultLibrary.makeFunction(name: "anchorGeometryFragmentLighting")! + geometryVertexDescriptor = MTLVertexDescriptor() + geometryVertexDescriptor.attributes[0].format = .float3 + geometryVertexDescriptor.attributes[0].offset = 0 + geometryVertexDescriptor.attributes[0].bufferIndex = 0 + geometryVertexDescriptor.attributes[1].format = .float2 + geometryVertexDescriptor.attributes[1].offset = 0 + geometryVertexDescriptor.attributes[1].bufferIndex = 1 + geometryVertexDescriptor.attributes[2].format = .half3 + geometryVertexDescriptor.attributes[2].offset = 8 + geometryVertexDescriptor.attributes[2].bufferIndex = 1 + geometryVertexDescriptor.layouts[0].stride = 12 + geometryVertexDescriptor.layouts[0].stepRate = 1 + geometryVertexDescriptor.layouts[0].stepFunction = .perVertex + geometryVertexDescriptor.layouts[1].stride = 16 + geometryVertexDescriptor.layouts[1].stepRate = 1 + geometryVertexDescriptor.layouts[1].stepFunction = .perVertex + let anchorPipelineStateDescriptor = MTLRenderPipelineDescriptor() + anchorPipelineStateDescriptor.label = "MyAnchorPipeline" + anchorPipelineStateDescriptor.sampleCount = renderDestination.sampleCount + anchorPipelineStateDescriptor.vertexFunction = anchorGeometryVertexFunction + anchorPipelineStateDescriptor.fragmentFunction = anchorGeometryFragmentFunction + anchorPipelineStateDescriptor.vertexDescriptor = geometryVertexDescriptor + anchorPipelineStateDescriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat + anchorPipelineStateDescriptor.depthAttachmentPixelFormat = renderDestination.depthStencilPixelFormat + anchorPipelineStateDescriptor.stencilAttachmentPixelFormat = renderDestination.depthStencilPixelFormat + do { try anchorPipelineState = device.makeRenderPipelineState(descriptor: anchorPipelineStateDescriptor) } + catch let error { print("Failed to created anchor geometry pipeline state, error \(error)") } + let anchorDepthStateDescriptor = MTLDepthStencilDescriptor() + anchorDepthStateDescriptor.depthCompareFunction = .less + anchorDepthStateDescriptor.isDepthWriteEnabled = true + anchorDepthState = device.makeDepthStencilState(descriptor: anchorDepthStateDescriptor) + commandQueue = device.makeCommandQueue() + } + + func setupAssets() { + let metalAllocator = MTKMeshBufferAllocator(device: device) + let vertexDescriptor = MTKModelIOVertexDescriptorFromMetal(geometryVertexDescriptor) + (vertexDescriptor.attributes[0] as! MDLVertexAttribute).name = MDLVertexAttributePosition + (vertexDescriptor.attributes[1] as! MDLVertexAttribute).name = MDLVertexAttributeTextureCoordinate + (vertexDescriptor.attributes[2] as! MDLVertexAttribute).name = MDLVertexAttributeNormal + let mdlMesh = MDLMesh(boxWithExtent: vector3(0.075, 0.075, 0.075), segments: vector3(1, 1, 1), inwardNormals: false, geometryType: .triangles, allocator: metalAllocator) + mdlMesh.vertexDescriptor = vertexDescriptor + do { try mesh = MTKMesh(mesh: mdlMesh, device: device) } + catch let error { print("Error creating MetalKit mesh, error \(error)") } + } + + func drawRectResized(size: CGSize) { + viewportSize = size + viewportSizeDidChange = true + } + + func update() { + let _ = inFlightSemaphore.wait(timeout: DispatchTime.distantFuture) + let commandBuffer = commandQueue.makeCommandBuffer() + commandBuffer.addCompletedHandler{ [weak self] commandBuffer in + if let strongSelf = self { strongSelf.inFlightSemaphore.signal() } + return + } + updateBufferStates() + updateGameState() + guard let passDescriptor = renderDestination.currentRenderPassDescriptor, + let drawable = renderDestination.currentDrawable else { return } + let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: passDescriptor) + drawCapturedImage(renderEncoder: renderEncoder) + drawAnchorGeometry(renderEncoder: renderEncoder) + renderEncoder.endEncoding() + commandBuffer.present(drawable) + commandBuffer.commit() + } + + func updateBufferStates() { + uniformBufferIndex = (uniformBufferIndex + 1) % maxBuffersInFlight + sharedUniformBufferOffset = alignedSharedUniformSize * uniformBufferIndex + anchorUniformBufferOffset = alignedInstanceUniformSize * uniformBufferIndex + sharedUniformBufferAddress = sharedUniformBuffer.contents().advanced(by: sharedUniformBufferOffset) + anchorUniformBufferAddress = anchorUniformBuffer.contents().advanced(by: anchorUniformBufferOffset) + } + + func updateGameState() { + guard let currentFrame = session.currentFrame else { return } + updateSharedUniforms(frame: currentFrame) + updateAnchors(frame: currentFrame) + updateCapturedImageTextures(frame: currentFrame) + if viewportSizeDidChange { + viewportSizeDidChange = false + updateImagePlane(frame: currentFrame) + } + } + + func updateSharedUniforms(frame: ARFrame) { + let uniforms = sharedUniformBufferAddress.assumingMemoryBound(to: SharedUniforms.self) + uniforms.pointee.viewMatrix = simd_inverse(frame.camera.transform) + uniforms.pointee.projectionMatrix = frame.camera.projectionMatrix(withViewportSize: viewportSize, orientation: .landscapeRight, zNear: 0.001, zFar: 1000) + var ambientIntensity: Float = 1.0 + if let lightEstimate = frame.lightEstimate { + ambientIntensity = Float(lightEstimate.ambientIntensity) / 1000.0 + } + let ambientLightColor: vector_float3 = vector3(0.5, 0.5, 0.5) + uniforms.pointee.ambientLightColor = ambientLightColor * ambientIntensity + var directionalLightDirection : vector_float3 = vector3(0.0, 0.0, -1.0) + directionalLightDirection = simd_normalize(directionalLightDirection) + uniforms.pointee.directionalLightDirection = directionalLightDirection + let directionalLightColor: vector_float3 = vector3(0.6, 0.6, 0.6) + uniforms.pointee.directionalLightColor = directionalLightColor * ambientIntensity + uniforms.pointee.materialShininess = 30 + } + + func updateAnchors(frame: ARFrame) { + anchorInstanceCount = min(frame.anchors.count, maxAnchorInstanceCount) + var anchorOffset: Int = 0 + if anchorInstanceCount == maxAnchorInstanceCount { + anchorOffset = max(frame.anchors.count - maxAnchorInstanceCount, 0) + } + for index in 0.. MTLTexture? { + var mtlTexture: MTLTexture? = nil + let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex) + let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex) + var texture: CVMetalTexture? = nil + let status = CVMetalTextureCacheCreateTextureFromImage(nil, capturedImageTextureCache, pixelBuffer, nil, pixelFormat, width, height, planeIndex, &texture) + if status == kCVReturnSuccess { mtlTexture = CVMetalTextureGetTexture(texture!) } + return mtlTexture + } + + func updateImagePlane(frame: ARFrame) { + let displayToCameraTransform = frame.displayTransform(withViewportSize: viewportSize, orientation: .landscapeRight).inverted() + let vertexData = imagePlaneVertexBuffer.contents().assumingMemoryBound(to: Float.self) + for index in 0...3 { + let textureCoordIndex = 4 * index + 2 + let textureCoord = CGPoint(x: CGFloat(planeVertexData[textureCoordIndex]), y: CGFloat(planeVertexData[textureCoordIndex + 1])) + let transformedCoord = textureCoord.applying(displayToCameraTransform) + vertexData[textureCoordIndex] = Float(transformedCoord.x) + vertexData[textureCoordIndex + 1] = Float(transformedCoord.y) + } + } + + func drawCapturedImage(renderEncoder: MTLRenderCommandEncoder) { + guard capturedImageTextureY != nil && capturedImageTextureCbCr != nil else { return } + renderEncoder.pushDebugGroup("DrawCapturedImage") + renderEncoder.setCullMode(.none) + renderEncoder.setRenderPipelineState(capturedImagePipelineState) + renderEncoder.setDepthStencilState(capturedImageDepthState) + renderEncoder.setVertexBuffer(imagePlaneVertexBuffer, offset: 0, index: 0) + renderEncoder.setFragmentTexture(capturedImageTextureY, index: 1) + renderEncoder.setFragmentTexture(capturedImageTextureCbCr, index: 2) + renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + renderEncoder.popDebugGroup() + } + + func drawAnchorGeometry(renderEncoder: MTLRenderCommandEncoder) { + guard anchorInstanceCount > 0 else { return } + renderEncoder.pushDebugGroup("DrawAnchors") + renderEncoder.setCullMode(.back) + renderEncoder.setRenderPipelineState(anchorPipelineState) + renderEncoder.setDepthStencilState(anchorDepthState) + renderEncoder.setVertexBuffer(anchorUniformBuffer, offset: anchorUniformBufferOffset, index: 2) + renderEncoder.setVertexBuffer(sharedUniformBuffer, offset: sharedUniformBufferOffset, index: 3) + renderEncoder.setFragmentBuffer(sharedUniformBuffer, offset: sharedUniformBufferOffset, index: 3) + for bufferIndex in 0.. +using namespace metal; + +typedef struct { + float2 position [[attribute(0)]]; + float2 texCoord [[attribute(1)]]; +} ImageVertex; + +typedef struct { + float4 position [[position]]; + float2 texCoord; +} ImageColorInOut; + +typedef struct { + float4x4 projectionMatrix; + float4x4 viewMatrix; + float3 ambientLightColor; + float3 directionalLightDirection; + float3 directionalLightColor; + float materialShininess; +} SharedUniforms; + +typedef struct { + float4x4 modelMatrix; +} InstanceUniforms; + +vertex ImageColorInOut capturedImageVertexTransform(ImageVertex in [[stage_in]]) { + ImageColorInOut out; + out.position = float4(in.position, 0.0, 1.0); + out.texCoord = in.texCoord; + return out; +} + +fragment float4 capturedImageFragmentShader(ImageColorInOut in [[stage_in]], + texture2d textureY [[ texture(1) ]], + texture2d textureCbCr [[ texture(2) ]]) { + constexpr sampler colorSampler(mip_filter::linear, mag_filter::linear, min_filter::linear); + const float4x4 ycbcrToRGBTransform = float4x4(float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f), + float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f), + float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f), + float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f)); + float4 ycbcr = float4(textureY.sample(colorSampler, in.texCoord).r, textureCbCr.sample(colorSampler, in.texCoord).rg, 1.0); + return ycbcrToRGBTransform * ycbcr; +} + +typedef struct { + float3 position [[attribute(0)]]; + float2 texCoord [[attribute(1)]]; + half3 normal [[attribute(2)]]; +} Vertex; + +typedef struct { + float4 position [[position]]; + float4 color; + half3 eyePosition; + half3 normal; +} ColorInOut; + +vertex ColorInOut anchorGeometryVertexTransform(Vertex in [[stage_in]], + constant SharedUniforms &sharedUniforms [[ buffer(3) ]], + constant InstanceUniforms *instanceUniforms [[ buffer(2) ]], + ushort vid [[vertex_id]], + ushort iid [[instance_id]]) { + ColorInOut out; + float4 position = float4(in.position, 1.0); + float4x4 modelMatrix = instanceUniforms[iid].modelMatrix; + float4x4 modelViewMatrix = sharedUniforms.viewMatrix * modelMatrix; + out.position = sharedUniforms.projectionMatrix * modelViewMatrix * position; + ushort colorID = vid / 4 % 6; + out.color = colorID == 0 ? float4(0.0, 1.0, 0.0, 1.0) // Right face + : colorID == 1 ? float4(1.0, 0.0, 0.0, 1.0) // Left face + : colorID == 2 ? float4(0.0, 0.0, 1.0, 1.0) // Top face + : colorID == 3 ? float4(1.0, 0.5, 0.0, 1.0) // Bottom face + : colorID == 4 ? float4(1.0, 1.0, 0.0, 1.0) // Back face + : float4(1.0, 1.0, 1.0, 1.0); // Front face + out.eyePosition = half3((modelViewMatrix * position).xyz); + float4 normal = modelMatrix * float4(in.normal.x, in.normal.y, in.normal.z, 0.0f); + out.normal = normalize(half3(normal.xyz)); + return out; +} + +fragment float4 anchorGeometryFragmentLighting(ColorInOut in [[stage_in]], + constant SharedUniforms &uniforms [[ buffer(3) ]]) { + float3 normal = float3(in.normal); + float3 directionalContribution = float3(0); + { + float nDotL = saturate(dot(normal, -uniforms.directionalLightDirection)); + float3 diffuseTerm = uniforms.directionalLightColor * nDotL; + float3 halfwayVector = normalize(-uniforms.directionalLightDirection - float3(in.eyePosition)); + float reflectionAngle = saturate(dot(normal, halfwayVector)); + float specularIntensity = saturate(powr(reflectionAngle, uniforms.materialShininess)); + float3 specularTerm = uniforms.directionalLightColor * specularIntensity; + directionalContribution = diffuseTerm + specularTerm; + } + float3 ambientContribution = uniforms.ambientLightColor; + float3 lightContributions = ambientContribution + directionalContribution; + float3 color = in.color.rgb * lightContributions; + return float4(color, in.color.w); +} diff --git a/ARKit/arkitdemo/ViewController.swift b/ARKit/arkitdemo/ViewController.swift new file mode 100644 index 0000000..5a5ae5e --- /dev/null +++ b/ARKit/arkitdemo/ViewController.swift @@ -0,0 +1,61 @@ + +import MetalKit +import ARKit + +extension MTKView : RenderDestinationProvider {} + +class ViewController: UIViewController, MTKViewDelegate, ARSessionDelegate { + + var session: ARSession! + var renderer: Renderer! + + override func viewDidLoad() { + super.viewDidLoad() + session = ARSession() + session.delegate = self + if let view = self.view as? MTKView { + view.device = MTLCreateSystemDefaultDevice() + view.delegate = self + renderer = Renderer(session: session, metalDevice: view.device!, renderDestination: view) + renderer.drawRectResized(size: view.bounds.size) + } + let tapGesture = UITapGestureRecognizer(target: self, action: #selector(ViewController.handleTap(gestureRecognize:))) + view.addGestureRecognizer(tapGesture) + } + + override func viewWillAppear(_ animated: Bool) { + super.viewWillAppear(animated) + let configuration = ARWorldTrackingSessionConfiguration() + session.run(configuration) + } + + override func viewWillDisappear(_ animated: Bool) { + super.viewWillDisappear(animated) + session.pause() + } + + @objc + func handleTap(gestureRecognize: UITapGestureRecognizer) { + if let currentFrame = session.currentFrame { + var translation = matrix_identity_float4x4 + translation.columns.3.z = -0.2 + let transform = simd_mul(currentFrame.camera.transform, translation) + let anchor = ARAnchor(transform: transform) + session.add(anchor: anchor) + } + } + + func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) { + renderer.drawRectResized(size: size) + } + + func draw(in view: MTKView) { + renderer.update() + } + + func session(_ session: ARSession, didFailWithError error: Error) {} + + func sessionWasInterrupted(_ session: ARSession) {} + + func sessionInterruptionEnded(_ session: ARSession) {} +} diff --git a/README.md b/README.md index b3a760a..0d1e666 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,4 @@ Repository to accompany the following blog posts: - [Working with memory in Metal](http://metalkit.org/2017/04/30/working-with-memory-in-metal.html) - [Working with memory in Metal part 2](http://metalkit.org/2017/05/26/working-with-memory-in-metal-part-2.html) - [Introducing Metal 2](http://metalkit.org/2017/06/30/introducing-metal-2.html) +- [Using ARKit with Metal](http://metalkit.org/2017/07/29/using-arkit-with-metal.html)