added arkit metal

This commit is contained in:
Marius Horga 2017-07-29 00:17:49 -05:00
parent ae2a70c7b7
commit 840c14d583
13 changed files with 883 additions and 0 deletions

View File

@ -0,0 +1,309 @@
// !$*UTF8*$!
{
archiveVersion = 1;
classes = {
};
objectVersion = 48;
objects = {
/* Begin PBXBuildFile section */
3C335AC71F27D12500FF39A5 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C335AC61F27D12500FF39A5 /* AppDelegate.swift */; };
3C335AC91F27D12500FF39A5 /* Shaders.metal in Sources */ = {isa = PBXBuildFile; fileRef = 3C335AC81F27D12500FF39A5 /* Shaders.metal */; };
3C335ACC1F27D12500FF39A5 /* Renderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C335ACB1F27D12500FF39A5 /* Renderer.swift */; };
3C335ACE1F27D12500FF39A5 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3C335ACD1F27D12500FF39A5 /* ViewController.swift */; };
3C335AD11F27D12500FF39A5 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 3C335ACF1F27D12500FF39A5 /* Main.storyboard */; };
/* End PBXBuildFile section */
/* Begin PBXFileReference section */
3C335AC31F27D12500FF39A5 /* arkitdemo.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = arkitdemo.app; sourceTree = BUILT_PRODUCTS_DIR; };
3C335AC61F27D12500FF39A5 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
3C335AC81F27D12500FF39A5 /* Shaders.metal */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.metal; path = Shaders.metal; sourceTree = "<group>"; };
3C335ACB1F27D12500FF39A5 /* Renderer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Renderer.swift; sourceTree = "<group>"; };
3C335ACD1F27D12500FF39A5 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
3C335AD01F27D12500FF39A5 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
3C335AD71F27D12500FF39A5 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
3C335AC01F27D12500FF39A5 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
3C335ABA1F27D12500FF39A5 = {
isa = PBXGroup;
children = (
3C335AC51F27D12500FF39A5 /* arkitdemo */,
3C335AC41F27D12500FF39A5 /* Products */,
);
sourceTree = "<group>";
};
3C335AC41F27D12500FF39A5 /* Products */ = {
isa = PBXGroup;
children = (
3C335AC31F27D12500FF39A5 /* arkitdemo.app */,
);
name = Products;
sourceTree = "<group>";
};
3C335AC51F27D12500FF39A5 /* arkitdemo */ = {
isa = PBXGroup;
children = (
3C335AC81F27D12500FF39A5 /* Shaders.metal */,
3C335ACB1F27D12500FF39A5 /* Renderer.swift */,
3C335ACD1F27D12500FF39A5 /* ViewController.swift */,
3C335ACF1F27D12500FF39A5 /* Main.storyboard */,
3C335AD71F27D12500FF39A5 /* Info.plist */,
3C335AC61F27D12500FF39A5 /* AppDelegate.swift */,
);
path = arkitdemo;
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXNativeTarget section */
3C335AC21F27D12500FF39A5 /* arkitdemo */ = {
isa = PBXNativeTarget;
buildConfigurationList = 3C335ADB1F27D12500FF39A5 /* Build configuration list for PBXNativeTarget "arkitdemo" */;
buildPhases = (
3C335ABF1F27D12500FF39A5 /* Sources */,
3C335AC01F27D12500FF39A5 /* Frameworks */,
3C335AC11F27D12500FF39A5 /* Resources */,
);
buildRules = (
);
dependencies = (
);
name = arkitdemo;
productName = arkitdemo;
productReference = 3C335AC31F27D12500FF39A5 /* arkitdemo.app */;
productType = "com.apple.product-type.application";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
3C335ABB1F27D12500FF39A5 /* Project object */ = {
isa = PBXProject;
attributes = {
LastSwiftUpdateCheck = 0900;
LastUpgradeCheck = 0900;
ORGANIZATIONNAME = "Marius Horga";
TargetAttributes = {
3C335AC21F27D12500FF39A5 = {
CreatedOnToolsVersion = 9.0;
};
};
};
buildConfigurationList = 3C335ABE1F27D12500FF39A5 /* Build configuration list for PBXProject "arkitdemo" */;
compatibilityVersion = "Xcode 8.0";
developmentRegion = en;
hasScannedForEncodings = 0;
knownRegions = (
en,
Base,
);
mainGroup = 3C335ABA1F27D12500FF39A5;
productRefGroup = 3C335AC41F27D12500FF39A5 /* Products */;
projectDirPath = "";
projectRoot = "";
targets = (
3C335AC21F27D12500FF39A5 /* arkitdemo */,
);
};
/* End PBXProject section */
/* Begin PBXResourcesBuildPhase section */
3C335AC11F27D12500FF39A5 /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
3C335AD11F27D12500FF39A5 /* Main.storyboard in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXResourcesBuildPhase section */
/* Begin PBXSourcesBuildPhase section */
3C335ABF1F27D12500FF39A5 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
3C335AC91F27D12500FF39A5 /* Shaders.metal in Sources */,
3C335ACC1F27D12500FF39A5 /* Renderer.swift in Sources */,
3C335ACE1F27D12500FF39A5 /* ViewController.swift in Sources */,
3C335AC71F27D12500FF39A5 /* AppDelegate.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin PBXVariantGroup section */
3C335ACF1F27D12500FF39A5 /* Main.storyboard */ = {
isa = PBXVariantGroup;
children = (
3C335AD01F27D12500FF39A5 /* Base */,
);
name = Main.storyboard;
sourceTree = "<group>";
};
/* End PBXVariantGroup section */
/* Begin XCBuildConfiguration section */
3C335AD91F27D12500FF39A5 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
CODE_SIGN_IDENTITY = "iPhone Developer";
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
GCC_C_LANGUAGE_STANDARD = gnu11;
GCC_DYNAMIC_NO_PIC = NO;
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
"$(inherited)",
);
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
};
name = Debug;
};
3C335ADA1F27D12500FF39A5 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
CODE_SIGN_IDENTITY = "iPhone Developer";
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
GCC_C_LANGUAGE_STANDARD = gnu11;
GCC_NO_COMMON_BLOCKS = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 11.0;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule";
VALIDATE_PRODUCT = YES;
};
name = Release;
};
3C335ADC1F27D12500FF39A5 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
DEVELOPMENT_TEAM = C2X9Q6VFVJ;
INFOPLIST_FILE = arkitdemo/Info.plist;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
PRODUCT_BUNDLE_IDENTIFIER = org.metalkit.arkitdemo;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_VERSION = 4.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Debug;
};
3C335ADD1F27D12500FF39A5 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
DEVELOPMENT_TEAM = C2X9Q6VFVJ;
INFOPLIST_FILE = arkitdemo/Info.plist;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
PRODUCT_BUNDLE_IDENTIFIER = org.metalkit.arkitdemo;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_VERSION = 4.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
3C335ABE1F27D12500FF39A5 /* Build configuration list for PBXProject "arkitdemo" */ = {
isa = XCConfigurationList;
buildConfigurations = (
3C335AD91F27D12500FF39A5 /* Debug */,
3C335ADA1F27D12500FF39A5 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
3C335ADB1F27D12500FF39A5 /* Build configuration list for PBXNativeTarget "arkitdemo" */ = {
isa = XCConfigurationList;
buildConfigurations = (
3C335ADC1F27D12500FF39A5 /* Debug */,
3C335ADD1F27D12500FF39A5 /* Release */,
);
defaultConfigurationIsVisible = 0;
};
/* End XCConfigurationList section */
};
rootObject = 3C335ABB1F27D12500FF39A5 /* Project object */;
}

View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<Workspace
version = "1.0">
<FileRef
location = "self:arkitdemo.xcodeproj">
</FileRef>
</Workspace>

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>SchemeUserState</key>
<dict>
<key>arkitdemo.xcscheme</key>
<dict>
<key>orderHint</key>
<integer>0</integer>
</dict>
</dict>
</dict>
</plist>

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>SchemeUserState</key>
<dict>
<key>arkitdemo.xcscheme</key>
<dict>
<key>orderHint</key>
<integer>0</integer>
</dict>
</dict>
</dict>
</plist>

View File

@ -0,0 +1,7 @@
import UIKit
@UIApplicationMain
class AppDelegate: UIResponder, UIApplicationDelegate {
var window: UIWindow?
}

View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="11134" systemVersion="15F34" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BV1-FR-VrT">
<dependencies>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="11106"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<scenes>
<!--Game View Controller-->
<scene sceneID="tXr-a1-R10">
<objects>
<viewController id="BV1-FR-VrT" customClass="ViewController" customModuleProvider="target" sceneMemberID="viewController">
<layoutGuides>
<viewControllerLayoutGuide type="top" id="8aa-yV-Osq"/>
<viewControllerLayoutGuide type="bottom" id="qHh-Mt-9TT"/>
</layoutGuides>
<view key="view" opaque="NO" contentMode="scaleToFill" id="3se-qz-xqx" customClass="MTKView">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
</view>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="SZV-WD-TEh" sceneMemberID="firstResponder"/>
</objects>
</scene>
</scenes>
</document>

View File

@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>NSCameraUsageDescription</key>
<string>This application will use the camera for Augmented Reality.</string>
<key>UILaunchStoryboardName</key>
<string>Main</string>
<key>UIMainStoryboardFile</key>
<string>Main</string>
<key>UIRequiredDeviceCapabilities</key>
<array>
<string>armv7</string>
<string>metal</string>
</array>
<key>UIRequiresFullScreen</key>
<true/>
<key>UIStatusBarHidden</key>
<true/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
</dict>
</plist>

View File

@ -0,0 +1,298 @@
import MetalKit
import ARKit
protocol RenderDestinationProvider {
var currentRenderPassDescriptor: MTLRenderPassDescriptor? { get }
var currentDrawable: CAMetalDrawable? { get }
var colorPixelFormat: MTLPixelFormat { get set }
var depthStencilPixelFormat: MTLPixelFormat { get set }
var sampleCount: Int { get set }
}
let maxBuffersInFlight: Int = 3
let maxAnchorInstanceCount: Int = 64
class Renderer {
let session: ARSession
let device: MTLDevice
var commandQueue: MTLCommandQueue!
let inFlightSemaphore = DispatchSemaphore(value: maxBuffersInFlight)
var renderDestination: RenderDestinationProvider
var sharedUniformBuffer: MTLBuffer!
var anchorUniformBuffer: MTLBuffer!
var imagePlaneVertexBuffer: MTLBuffer!
var capturedImagePipelineState: MTLRenderPipelineState!
var capturedImageDepthState: MTLDepthStencilState!
var anchorPipelineState: MTLRenderPipelineState!
var anchorDepthState: MTLDepthStencilState!
var capturedImageTextureY: MTLTexture!
var capturedImageTextureCbCr: MTLTexture!
var capturedImageTextureCache: CVMetalTextureCache!
var geometryVertexDescriptor: MTLVertexDescriptor!
var mesh: MTKMesh!
var uniformBufferIndex: Int = 0
var sharedUniformBufferOffset: Int = 0
var anchorUniformBufferOffset: Int = 0
var sharedUniformBufferAddress: UnsafeMutableRawPointer!
var anchorUniformBufferAddress: UnsafeMutableRawPointer!
var anchorInstanceCount: Int = 0
var viewportSize: CGSize = CGSize()
var viewportSizeDidChange: Bool = false
struct SharedUniforms {
var projectionMatrix: matrix_float4x4
var viewMatrix: matrix_float4x4
var ambientLightColor: vector_float3
var directionalLightDirection: vector_float3
var directionalLightColor: vector_float3
var materialShininess: Float
}
struct InstanceUniforms {
var modelMatrix: matrix_float4x4
}
let alignedSharedUniformSize = (MemoryLayout<SharedUniforms>.size & ~0xFF) + 0x100
let alignedInstanceUniformSize = ((MemoryLayout<InstanceUniforms>.size * maxAnchorInstanceCount) & ~0xFF) + 0x100
let planeVertexData: [Float] = [-1, -1, 0, 1,
1, -1, 1, 1,
-1, 1, 0, 0,
1, 1, 1, 0]
init(session: ARSession, metalDevice device: MTLDevice, renderDestination: RenderDestinationProvider) {
self.session = session
self.device = device
self.renderDestination = renderDestination
setupPipeline()
setupAssets()
}
func setupPipeline() {
renderDestination.depthStencilPixelFormat = .depth32Float_stencil8
renderDestination.colorPixelFormat = .bgra8Unorm
renderDestination.sampleCount = 1
let sharedUniformBufferSize = alignedSharedUniformSize * maxBuffersInFlight
let anchorUniformBufferSize = alignedInstanceUniformSize * maxBuffersInFlight
sharedUniformBuffer = device.makeBuffer(length: sharedUniformBufferSize, options: .storageModeShared)
anchorUniformBuffer = device.makeBuffer(length: anchorUniformBufferSize, options: .storageModeShared)
let imagePlaneVertexDataCount = planeVertexData.count * MemoryLayout<Float>.size
imagePlaneVertexBuffer = device.makeBuffer(bytes: planeVertexData, length: imagePlaneVertexDataCount, options: [])
let defaultLibrary = device.makeDefaultLibrary()!
let capturedImageVertexFunction = defaultLibrary.makeFunction(name: "capturedImageVertexTransform")!
let capturedImageFragmentFunction = defaultLibrary.makeFunction(name: "capturedImageFragmentShader")!
let imagePlaneVertexDescriptor = MTLVertexDescriptor()
imagePlaneVertexDescriptor.attributes[0].format = .float2
imagePlaneVertexDescriptor.attributes[0].offset = 0
imagePlaneVertexDescriptor.attributes[0].bufferIndex = 0
imagePlaneVertexDescriptor.attributes[1].format = .float2
imagePlaneVertexDescriptor.attributes[1].offset = 8
imagePlaneVertexDescriptor.attributes[1].bufferIndex = 0
imagePlaneVertexDescriptor.layouts[0].stride = 16
imagePlaneVertexDescriptor.layouts[0].stepRate = 1
imagePlaneVertexDescriptor.layouts[0].stepFunction = .perVertex
let capturedImagePipelineStateDescriptor = MTLRenderPipelineDescriptor()
capturedImagePipelineStateDescriptor.label = "MyCapturedImagePipeline"
capturedImagePipelineStateDescriptor.sampleCount = renderDestination.sampleCount
capturedImagePipelineStateDescriptor.vertexFunction = capturedImageVertexFunction
capturedImagePipelineStateDescriptor.fragmentFunction = capturedImageFragmentFunction
capturedImagePipelineStateDescriptor.vertexDescriptor = imagePlaneVertexDescriptor
capturedImagePipelineStateDescriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat
capturedImagePipelineStateDescriptor.depthAttachmentPixelFormat = renderDestination.depthStencilPixelFormat
capturedImagePipelineStateDescriptor.stencilAttachmentPixelFormat = renderDestination.depthStencilPixelFormat
do { try capturedImagePipelineState = device.makeRenderPipelineState(descriptor: capturedImagePipelineStateDescriptor) }
catch let error { print("Failed to created captured image pipeline state, error \(error)") }
let capturedImageDepthStateDescriptor = MTLDepthStencilDescriptor()
capturedImageDepthStateDescriptor.depthCompareFunction = .always
capturedImageDepthStateDescriptor.isDepthWriteEnabled = false
capturedImageDepthState = device.makeDepthStencilState(descriptor: capturedImageDepthStateDescriptor)
var textureCache: CVMetalTextureCache?
CVMetalTextureCacheCreate(nil, nil, device, nil, &textureCache)
capturedImageTextureCache = textureCache
let anchorGeometryVertexFunction = defaultLibrary.makeFunction(name: "anchorGeometryVertexTransform")!
let anchorGeometryFragmentFunction = defaultLibrary.makeFunction(name: "anchorGeometryFragmentLighting")!
geometryVertexDescriptor = MTLVertexDescriptor()
geometryVertexDescriptor.attributes[0].format = .float3
geometryVertexDescriptor.attributes[0].offset = 0
geometryVertexDescriptor.attributes[0].bufferIndex = 0
geometryVertexDescriptor.attributes[1].format = .float2
geometryVertexDescriptor.attributes[1].offset = 0
geometryVertexDescriptor.attributes[1].bufferIndex = 1
geometryVertexDescriptor.attributes[2].format = .half3
geometryVertexDescriptor.attributes[2].offset = 8
geometryVertexDescriptor.attributes[2].bufferIndex = 1
geometryVertexDescriptor.layouts[0].stride = 12
geometryVertexDescriptor.layouts[0].stepRate = 1
geometryVertexDescriptor.layouts[0].stepFunction = .perVertex
geometryVertexDescriptor.layouts[1].stride = 16
geometryVertexDescriptor.layouts[1].stepRate = 1
geometryVertexDescriptor.layouts[1].stepFunction = .perVertex
let anchorPipelineStateDescriptor = MTLRenderPipelineDescriptor()
anchorPipelineStateDescriptor.label = "MyAnchorPipeline"
anchorPipelineStateDescriptor.sampleCount = renderDestination.sampleCount
anchorPipelineStateDescriptor.vertexFunction = anchorGeometryVertexFunction
anchorPipelineStateDescriptor.fragmentFunction = anchorGeometryFragmentFunction
anchorPipelineStateDescriptor.vertexDescriptor = geometryVertexDescriptor
anchorPipelineStateDescriptor.colorAttachments[0].pixelFormat = renderDestination.colorPixelFormat
anchorPipelineStateDescriptor.depthAttachmentPixelFormat = renderDestination.depthStencilPixelFormat
anchorPipelineStateDescriptor.stencilAttachmentPixelFormat = renderDestination.depthStencilPixelFormat
do { try anchorPipelineState = device.makeRenderPipelineState(descriptor: anchorPipelineStateDescriptor) }
catch let error { print("Failed to created anchor geometry pipeline state, error \(error)") }
let anchorDepthStateDescriptor = MTLDepthStencilDescriptor()
anchorDepthStateDescriptor.depthCompareFunction = .less
anchorDepthStateDescriptor.isDepthWriteEnabled = true
anchorDepthState = device.makeDepthStencilState(descriptor: anchorDepthStateDescriptor)
commandQueue = device.makeCommandQueue()
}
func setupAssets() {
let metalAllocator = MTKMeshBufferAllocator(device: device)
let vertexDescriptor = MTKModelIOVertexDescriptorFromMetal(geometryVertexDescriptor)
(vertexDescriptor.attributes[0] as! MDLVertexAttribute).name = MDLVertexAttributePosition
(vertexDescriptor.attributes[1] as! MDLVertexAttribute).name = MDLVertexAttributeTextureCoordinate
(vertexDescriptor.attributes[2] as! MDLVertexAttribute).name = MDLVertexAttributeNormal
let mdlMesh = MDLMesh(boxWithExtent: vector3(0.075, 0.075, 0.075), segments: vector3(1, 1, 1), inwardNormals: false, geometryType: .triangles, allocator: metalAllocator)
mdlMesh.vertexDescriptor = vertexDescriptor
do { try mesh = MTKMesh(mesh: mdlMesh, device: device) }
catch let error { print("Error creating MetalKit mesh, error \(error)") }
}
func drawRectResized(size: CGSize) {
viewportSize = size
viewportSizeDidChange = true
}
func update() {
let _ = inFlightSemaphore.wait(timeout: DispatchTime.distantFuture)
let commandBuffer = commandQueue.makeCommandBuffer()
commandBuffer.addCompletedHandler{ [weak self] commandBuffer in
if let strongSelf = self { strongSelf.inFlightSemaphore.signal() }
return
}
updateBufferStates()
updateGameState()
guard let passDescriptor = renderDestination.currentRenderPassDescriptor,
let drawable = renderDestination.currentDrawable else { return }
let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: passDescriptor)
drawCapturedImage(renderEncoder: renderEncoder)
drawAnchorGeometry(renderEncoder: renderEncoder)
renderEncoder.endEncoding()
commandBuffer.present(drawable)
commandBuffer.commit()
}
func updateBufferStates() {
uniformBufferIndex = (uniformBufferIndex + 1) % maxBuffersInFlight
sharedUniformBufferOffset = alignedSharedUniformSize * uniformBufferIndex
anchorUniformBufferOffset = alignedInstanceUniformSize * uniformBufferIndex
sharedUniformBufferAddress = sharedUniformBuffer.contents().advanced(by: sharedUniformBufferOffset)
anchorUniformBufferAddress = anchorUniformBuffer.contents().advanced(by: anchorUniformBufferOffset)
}
func updateGameState() {
guard let currentFrame = session.currentFrame else { return }
updateSharedUniforms(frame: currentFrame)
updateAnchors(frame: currentFrame)
updateCapturedImageTextures(frame: currentFrame)
if viewportSizeDidChange {
viewportSizeDidChange = false
updateImagePlane(frame: currentFrame)
}
}
func updateSharedUniforms(frame: ARFrame) {
let uniforms = sharedUniformBufferAddress.assumingMemoryBound(to: SharedUniforms.self)
uniforms.pointee.viewMatrix = simd_inverse(frame.camera.transform)
uniforms.pointee.projectionMatrix = frame.camera.projectionMatrix(withViewportSize: viewportSize, orientation: .landscapeRight, zNear: 0.001, zFar: 1000)
var ambientIntensity: Float = 1.0
if let lightEstimate = frame.lightEstimate {
ambientIntensity = Float(lightEstimate.ambientIntensity) / 1000.0
}
let ambientLightColor: vector_float3 = vector3(0.5, 0.5, 0.5)
uniforms.pointee.ambientLightColor = ambientLightColor * ambientIntensity
var directionalLightDirection : vector_float3 = vector3(0.0, 0.0, -1.0)
directionalLightDirection = simd_normalize(directionalLightDirection)
uniforms.pointee.directionalLightDirection = directionalLightDirection
let directionalLightColor: vector_float3 = vector3(0.6, 0.6, 0.6)
uniforms.pointee.directionalLightColor = directionalLightColor * ambientIntensity
uniforms.pointee.materialShininess = 30
}
func updateAnchors(frame: ARFrame) {
anchorInstanceCount = min(frame.anchors.count, maxAnchorInstanceCount)
var anchorOffset: Int = 0
if anchorInstanceCount == maxAnchorInstanceCount {
anchorOffset = max(frame.anchors.count - maxAnchorInstanceCount, 0)
}
for index in 0..<anchorInstanceCount {
let anchor = frame.anchors[index + anchorOffset]
var coordinateSpaceTransform = matrix_identity_float4x4
coordinateSpaceTransform.columns.2.z = -1.0 // flip Z axis to convert to left handed
let modelMatrix = simd_mul(anchor.transform, coordinateSpaceTransform)
let anchorUniforms = anchorUniformBufferAddress.assumingMemoryBound(to: InstanceUniforms.self).advanced(by: index)
anchorUniforms.pointee.modelMatrix = modelMatrix
}
}
func updateCapturedImageTextures(frame: ARFrame) {
let pixelBuffer = frame.capturedImage
if (CVPixelBufferGetPlaneCount(pixelBuffer) < 2) { return }
capturedImageTextureY = createTexture(fromPixelBuffer: pixelBuffer, pixelFormat:.r8Unorm, planeIndex:0)!
capturedImageTextureCbCr = createTexture(fromPixelBuffer: pixelBuffer, pixelFormat:.rg8Unorm, planeIndex:1)!
}
func createTexture(fromPixelBuffer pixelBuffer: CVPixelBuffer, pixelFormat: MTLPixelFormat, planeIndex: Int) -> MTLTexture? {
var mtlTexture: MTLTexture? = nil
let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex)
let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex)
var texture: CVMetalTexture? = nil
let status = CVMetalTextureCacheCreateTextureFromImage(nil, capturedImageTextureCache, pixelBuffer, nil, pixelFormat, width, height, planeIndex, &texture)
if status == kCVReturnSuccess { mtlTexture = CVMetalTextureGetTexture(texture!) }
return mtlTexture
}
func updateImagePlane(frame: ARFrame) {
let displayToCameraTransform = frame.displayTransform(withViewportSize: viewportSize, orientation: .landscapeRight).inverted()
let vertexData = imagePlaneVertexBuffer.contents().assumingMemoryBound(to: Float.self)
for index in 0...3 {
let textureCoordIndex = 4 * index + 2
let textureCoord = CGPoint(x: CGFloat(planeVertexData[textureCoordIndex]), y: CGFloat(planeVertexData[textureCoordIndex + 1]))
let transformedCoord = textureCoord.applying(displayToCameraTransform)
vertexData[textureCoordIndex] = Float(transformedCoord.x)
vertexData[textureCoordIndex + 1] = Float(transformedCoord.y)
}
}
func drawCapturedImage(renderEncoder: MTLRenderCommandEncoder) {
guard capturedImageTextureY != nil && capturedImageTextureCbCr != nil else { return }
renderEncoder.pushDebugGroup("DrawCapturedImage")
renderEncoder.setCullMode(.none)
renderEncoder.setRenderPipelineState(capturedImagePipelineState)
renderEncoder.setDepthStencilState(capturedImageDepthState)
renderEncoder.setVertexBuffer(imagePlaneVertexBuffer, offset: 0, index: 0)
renderEncoder.setFragmentTexture(capturedImageTextureY, index: 1)
renderEncoder.setFragmentTexture(capturedImageTextureCbCr, index: 2)
renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
renderEncoder.popDebugGroup()
}
func drawAnchorGeometry(renderEncoder: MTLRenderCommandEncoder) {
guard anchorInstanceCount > 0 else { return }
renderEncoder.pushDebugGroup("DrawAnchors")
renderEncoder.setCullMode(.back)
renderEncoder.setRenderPipelineState(anchorPipelineState)
renderEncoder.setDepthStencilState(anchorDepthState)
renderEncoder.setVertexBuffer(anchorUniformBuffer, offset: anchorUniformBufferOffset, index: 2)
renderEncoder.setVertexBuffer(sharedUniformBuffer, offset: sharedUniformBufferOffset, index: 3)
renderEncoder.setFragmentBuffer(sharedUniformBuffer, offset: sharedUniformBufferOffset, index: 3)
for bufferIndex in 0..<mesh.vertexBuffers.count {
let vertexBuffer = mesh.vertexBuffers[bufferIndex]
renderEncoder.setVertexBuffer(vertexBuffer.buffer, offset: vertexBuffer.offset, index:bufferIndex)
}
for submesh in mesh.submeshes {
renderEncoder.drawIndexedPrimitives(type: submesh.primitiveType, indexCount: submesh.indexCount, indexType: submesh.indexType, indexBuffer: submesh.indexBuffer.buffer, indexBufferOffset: submesh.indexBuffer.offset, instanceCount: anchorInstanceCount)
}
renderEncoder.popDebugGroup()
}
}

View File

@ -0,0 +1,100 @@
#include <metal_stdlib>
using namespace metal;
typedef struct {
float2 position [[attribute(0)]];
float2 texCoord [[attribute(1)]];
} ImageVertex;
typedef struct {
float4 position [[position]];
float2 texCoord;
} ImageColorInOut;
typedef struct {
float4x4 projectionMatrix;
float4x4 viewMatrix;
float3 ambientLightColor;
float3 directionalLightDirection;
float3 directionalLightColor;
float materialShininess;
} SharedUniforms;
typedef struct {
float4x4 modelMatrix;
} InstanceUniforms;
vertex ImageColorInOut capturedImageVertexTransform(ImageVertex in [[stage_in]]) {
ImageColorInOut out;
out.position = float4(in.position, 0.0, 1.0);
out.texCoord = in.texCoord;
return out;
}
fragment float4 capturedImageFragmentShader(ImageColorInOut in [[stage_in]],
texture2d<float, access::sample> textureY [[ texture(1) ]],
texture2d<float, access::sample> textureCbCr [[ texture(2) ]]) {
constexpr sampler colorSampler(mip_filter::linear, mag_filter::linear, min_filter::linear);
const float4x4 ycbcrToRGBTransform = float4x4(float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f),
float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f),
float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f),
float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f));
float4 ycbcr = float4(textureY.sample(colorSampler, in.texCoord).r, textureCbCr.sample(colorSampler, in.texCoord).rg, 1.0);
return ycbcrToRGBTransform * ycbcr;
}
typedef struct {
float3 position [[attribute(0)]];
float2 texCoord [[attribute(1)]];
half3 normal [[attribute(2)]];
} Vertex;
typedef struct {
float4 position [[position]];
float4 color;
half3 eyePosition;
half3 normal;
} ColorInOut;
vertex ColorInOut anchorGeometryVertexTransform(Vertex in [[stage_in]],
constant SharedUniforms &sharedUniforms [[ buffer(3) ]],
constant InstanceUniforms *instanceUniforms [[ buffer(2) ]],
ushort vid [[vertex_id]],
ushort iid [[instance_id]]) {
ColorInOut out;
float4 position = float4(in.position, 1.0);
float4x4 modelMatrix = instanceUniforms[iid].modelMatrix;
float4x4 modelViewMatrix = sharedUniforms.viewMatrix * modelMatrix;
out.position = sharedUniforms.projectionMatrix * modelViewMatrix * position;
ushort colorID = vid / 4 % 6;
out.color = colorID == 0 ? float4(0.0, 1.0, 0.0, 1.0) // Right face
: colorID == 1 ? float4(1.0, 0.0, 0.0, 1.0) // Left face
: colorID == 2 ? float4(0.0, 0.0, 1.0, 1.0) // Top face
: colorID == 3 ? float4(1.0, 0.5, 0.0, 1.0) // Bottom face
: colorID == 4 ? float4(1.0, 1.0, 0.0, 1.0) // Back face
: float4(1.0, 1.0, 1.0, 1.0); // Front face
out.eyePosition = half3((modelViewMatrix * position).xyz);
float4 normal = modelMatrix * float4(in.normal.x, in.normal.y, in.normal.z, 0.0f);
out.normal = normalize(half3(normal.xyz));
return out;
}
fragment float4 anchorGeometryFragmentLighting(ColorInOut in [[stage_in]],
constant SharedUniforms &uniforms [[ buffer(3) ]]) {
float3 normal = float3(in.normal);
float3 directionalContribution = float3(0);
{
float nDotL = saturate(dot(normal, -uniforms.directionalLightDirection));
float3 diffuseTerm = uniforms.directionalLightColor * nDotL;
float3 halfwayVector = normalize(-uniforms.directionalLightDirection - float3(in.eyePosition));
float reflectionAngle = saturate(dot(normal, halfwayVector));
float specularIntensity = saturate(powr(reflectionAngle, uniforms.materialShininess));
float3 specularTerm = uniforms.directionalLightColor * specularIntensity;
directionalContribution = diffuseTerm + specularTerm;
}
float3 ambientContribution = uniforms.ambientLightColor;
float3 lightContributions = ambientContribution + directionalContribution;
float3 color = in.color.rgb * lightContributions;
return float4(color, in.color.w);
}

View File

@ -0,0 +1,61 @@
import MetalKit
import ARKit
extension MTKView : RenderDestinationProvider {}
class ViewController: UIViewController, MTKViewDelegate, ARSessionDelegate {
var session: ARSession!
var renderer: Renderer!
override func viewDidLoad() {
super.viewDidLoad()
session = ARSession()
session.delegate = self
if let view = self.view as? MTKView {
view.device = MTLCreateSystemDefaultDevice()
view.delegate = self
renderer = Renderer(session: session, metalDevice: view.device!, renderDestination: view)
renderer.drawRectResized(size: view.bounds.size)
}
let tapGesture = UITapGestureRecognizer(target: self, action: #selector(ViewController.handleTap(gestureRecognize:)))
view.addGestureRecognizer(tapGesture)
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
let configuration = ARWorldTrackingSessionConfiguration()
session.run(configuration)
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
session.pause()
}
@objc
func handleTap(gestureRecognize: UITapGestureRecognizer) {
if let currentFrame = session.currentFrame {
var translation = matrix_identity_float4x4
translation.columns.3.z = -0.2
let transform = simd_mul(currentFrame.camera.transform, translation)
let anchor = ARAnchor(transform: transform)
session.add(anchor: anchor)
}
}
func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
renderer.drawRectResized(size: size)
}
func draw(in view: MTKView) {
renderer.update()
}
func session(_ session: ARSession, didFailWithError error: Error) {}
func sessionWasInterrupted(_ session: ARSession) {}
func sessionInterruptionEnded(_ session: ARSession) {}
}

View File

@ -27,3 +27,4 @@ Repository to accompany the following blog posts:
- [Working with memory in Metal](http://metalkit.org/2017/04/30/working-with-memory-in-metal.html)
- [Working with memory in Metal part 2](http://metalkit.org/2017/05/26/working-with-memory-in-metal-part-2.html)
- [Introducing Metal 2](http://metalkit.org/2017/06/30/introducing-metal-2.html)
- [Using ARKit with Metal](http://metalkit.org/2017/07/29/using-arkit-with-metal.html)