diff --git a/stdlib/public/core/Span/MutableSpan.swift b/stdlib/public/core/Span/MutableSpan.swift index 3650544fb380c..8f214f571fae6 100644 --- a/stdlib/public/core/Span/MutableSpan.swift +++ b/stdlib/public/core/Span/MutableSpan.swift @@ -236,6 +236,7 @@ extension MutableSpan where Element: ~Copyable { extension MutableSpan where Element: ~Copyable { @_alwaysEmitIntoClient + @_semantics("fixed_storage.get_count") public var count: Int { _assumeNonNegative(_count) } @_alwaysEmitIntoClient @@ -279,7 +280,13 @@ extension MutableSpan where Element: BitwiseCopyable { @available(SwiftCompatibilitySpan 5.0, *) @_originallyDefinedIn(module: "Swift;CompatibilitySpan", SwiftCompatibilitySpan 6.2) extension MutableSpan where Element: ~Copyable { - + // SILOptimizer looks for fixed_storage.check_index semantics for bounds check optimizations. + @_semantics("fixed_storage.check_index") + @inline(__always) + @_alwaysEmitIntoClient + internal func _checkIndex(_ position: Index) { + _precondition(indices.contains(position), "index out of bounds") + } /// Accesses the element at the specified position in the `MutableSpan`. /// /// - Parameter position: The offset of the element to access. `position` @@ -289,12 +296,12 @@ extension MutableSpan where Element: ~Copyable { @_alwaysEmitIntoClient public subscript(_ position: Index) -> Element { unsafeAddress { - _precondition(indices.contains(position), "index out of bounds") + _checkIndex(position) return unsafe UnsafePointer(_unsafeAddressOfElement(unchecked: position)) } @lifetime(self: copy self) unsafeMutableAddress { - _precondition(indices.contains(position), "index out of bounds") + _checkIndex(position) return unsafe _unsafeAddressOfElement(unchecked: position) } } diff --git a/stdlib/public/core/Span/OutputSpan.swift b/stdlib/public/core/Span/OutputSpan.swift index 41ae4d7f7bef4..50079f8321863 100644 --- a/stdlib/public/core/Span/OutputSpan.swift +++ b/stdlib/public/core/Span/OutputSpan.swift @@ -82,6 +82,7 @@ extension OutputSpan where Element: ~Copyable { extension OutputSpan where Element: ~Copyable { /// The number of initialized elements in this span. @_alwaysEmitIntoClient + @_semantics("fixed_storage.get_count") public var count: Int { _count } /// The number of additional elements that can be added to this span. @@ -193,6 +194,14 @@ extension OutputSpan where Element: ~Copyable { unsafe Range(_uncheckedBounds: (0, _count)) } + // SILOptimizer looks for fixed_storage.check_index semantics for bounds check optimizations. + @_semantics("fixed_storage.check_index") + @inline(__always) + @_alwaysEmitIntoClient + internal func _checkIndex(_ index: Index) { + _precondition(indices.contains(index), "Index out of bounds") + } + /// Accesses the element at the specified position. /// /// - Parameter index: A valid index into this span. @@ -201,12 +210,12 @@ extension OutputSpan where Element: ~Copyable { @_alwaysEmitIntoClient public subscript(_ index: Index) -> Element { unsafeAddress { - _precondition(indices.contains(index), "index out of bounds") + _checkIndex(index) return unsafe UnsafePointer(_unsafeAddressOfElement(unchecked: index)) } @lifetime(self: copy self) unsafeMutableAddress { - _precondition(indices.contains(index), "index out of bounds") + _checkIndex(index) return unsafe _unsafeAddressOfElement(unchecked: index) } } diff --git a/stdlib/public/core/Span/Span.swift b/stdlib/public/core/Span/Span.swift index fd930aea00706..82e774115fc43 100644 --- a/stdlib/public/core/Span/Span.swift +++ b/stdlib/public/core/Span/Span.swift @@ -414,6 +414,7 @@ extension Span where Element: ~Copyable { @available(SwiftCompatibilitySpan 5.0, *) @_originallyDefinedIn(module: "Swift;CompatibilitySpan", SwiftCompatibilitySpan 6.2) extension Span where Element: ~Copyable { + // SILOptimizer looks for fixed_storage.check_index semantics for bounds check optimizations. @_semantics("fixed_storage.check_index") @inline(__always) @_alwaysEmitIntoClient diff --git a/test/SILOptimizer/Inputs/SpanExtras.swift b/test/SILOptimizer/Inputs/SpanExtras.swift deleted file mode 100644 index c591c4dc337f5..0000000000000 --- a/test/SILOptimizer/Inputs/SpanExtras.swift +++ /dev/null @@ -1,1027 +0,0 @@ -import Builtin - -// A MutableSpan represents a span of memory which -// contains initialized `Element` instances. -@frozen -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -public struct MutableSpan: ~Copyable & ~Escapable { - @usableFromInline let _pointer: UnsafeMutableRawPointer? - - @usableFromInline let _count: Int - - @_alwaysEmitIntoClient - internal func _start() -> UnsafeMutableRawPointer { - _pointer.unsafelyUnwrapped - } - - @usableFromInline @inline(__always) - @_lifetime(borrow start) - init( - _unchecked start: UnsafeMutableRawPointer?, - count: Int - ) { - _pointer = start - _count = count - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan: @unchecked Sendable where Element: Sendable {} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: ~Copyable { - - @_alwaysEmitIntoClient - @usableFromInline @inline(__always) - @_lifetime(borrow elements) - internal init( - _unchecked elements: UnsafeMutableBufferPointer - ) { - _pointer = .init(elements.baseAddress) - _count = elements.count - } - - @_alwaysEmitIntoClient - @_lifetime(borrow buffer) - public init( - _unsafeElements buffer: UnsafeMutableBufferPointer - ) { - precondition( - ((Int(bitPattern: buffer.baseAddress) & - (MemoryLayout.alignment&-1)) == 0), - "baseAddress must be properly aligned to access Element" - ) - let ms = MutableSpan(_unchecked: buffer) - self = _overrideLifetime(ms, borrowing: buffer) - } - - @_alwaysEmitIntoClient - @_lifetime(borrow start) - public init( - _unsafeStart start: UnsafeMutablePointer, - count: Int - ) { - precondition(count >= 0, "Count must not be negative") - let buffer = UnsafeMutableBufferPointer(start: start, count: count) - let ms = MutableSpan(_unsafeElements: buffer) - self = _overrideLifetime(ms, borrowing: start) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan { - - @_alwaysEmitIntoClient - @_lifetime(borrow elements) - public init( - _unsafeElements elements: borrowing Slice> - ) { - let rb = UnsafeMutableBufferPointer(rebasing: elements) - let ms = MutableSpan(_unsafeElements: rb) - self = _overrideLifetime(ms, borrowing: elements) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: BitwiseCopyable { - - @_alwaysEmitIntoClient - @_lifetime(borrow buffer) - public init( - _unsafeBytes buffer: UnsafeMutableRawBufferPointer - ) { - precondition( - ((Int(bitPattern: buffer.baseAddress) & - (MemoryLayout.alignment&-1)) == 0), - "baseAddress must be properly aligned to access Element" - ) - let (byteCount, stride) = (buffer.count, MemoryLayout.stride) - let (count, remainder) = byteCount.quotientAndRemainder(dividingBy: stride) - precondition(remainder == 0, "Span must contain a whole number of elements") - let elements = UnsafeMutableBufferPointer( - start: buffer.baseAddress?.assumingMemoryBound(to: Element.self), - count: count - ) - let ms = MutableSpan(_unsafeElements: elements) - self = _overrideLifetime(ms, borrowing: buffer) - } - - @_alwaysEmitIntoClient - @_lifetime(borrow pointer) - public init( - _unsafeStart pointer: UnsafeMutableRawPointer, - byteCount: Int - ) { - precondition(byteCount >= 0, "Count must not be negative") - let bytes = UnsafeMutableRawBufferPointer(start: pointer, count: byteCount) - let ms = MutableSpan(_unsafeBytes: bytes) - self = _overrideLifetime(ms, borrowing: pointer) - } - - @_alwaysEmitIntoClient - @_lifetime(borrow buffer) - public init( - _unsafeBytes buffer: borrowing Slice - ) { - let bytes = UnsafeMutableRawBufferPointer(rebasing: buffer) - let ms = MutableSpan(_unsafeBytes: bytes) - self = _overrideLifetime(ms, borrowing: buffer) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension Span where Element: ~Copyable { - - @_alwaysEmitIntoClient - @_lifetime(borrow mutableSpan) - public init(_unsafeMutableSpan mutableSpan: borrowing MutableSpan) { - let pointer = mutableSpan._pointer?.assumingMemoryBound(to: Element.self) - let buffer = UnsafeBufferPointer(start: pointer, count: mutableSpan.count) - let span = Span(_unsafeElements: buffer) - self = _overrideLifetime(span, borrowing: mutableSpan) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: ~Copyable { - - @_alwaysEmitIntoClient - public var storage: Span { - @_lifetime(borrow self) - borrowing get { - Span(_unsafeMutableSpan: self) - } - } - - @_alwaysEmitIntoClient - public func withSpan( - _ body: (Span) throws(E) -> Result - ) throws(E) -> Result { - try body(Span(_unsafeMutableSpan: self)) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension RawSpan { - - @_alwaysEmitIntoClient - @_lifetime(borrow mutableSpan) - public init( - _unsafeMutableSpan mutableSpan: borrowing MutableSpan - ) { - let pointer = mutableSpan._pointer - let byteCount = mutableSpan.count &* MemoryLayout.stride - let buffer = UnsafeRawBufferPointer(start: pointer, count: byteCount) - let rawSpan = RawSpan(_unsafeBytes: buffer) - self = _overrideLifetime(rawSpan, borrowing: mutableSpan) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: Equatable { - - @_alwaysEmitIntoClient - public func _elementsEqual(_ other: borrowing Self) -> Bool { - _elementsEqual(Span(_unsafeMutableSpan: other)) - } - - @_alwaysEmitIntoClient - public func _elementsEqual(_ other: Span) -> Bool { - Span(_unsafeMutableSpan: self)._elementsEqual(other) - } - - @_alwaysEmitIntoClient - public func _elementsEqual(_ other: some Collection) -> Bool { - Span(_unsafeMutableSpan: self)._elementsEqual(other) - } - - @_alwaysEmitIntoClient - public func _elementsEqual(_ other: some Sequence) -> Bool { - Span(_unsafeMutableSpan: self)._elementsEqual(other) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: ~Copyable { - - @_alwaysEmitIntoClient - public var _description: String { - let addr = String(UInt(bitPattern: _pointer), radix: 16, uppercase: false) - return "(0x\(addr), \(_count))" - } -} - -//MARK: Collection, RandomAccessCollection -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: ~Copyable { - - @_alwaysEmitIntoClient - public var count: Int { _count } - - @_alwaysEmitIntoClient - public var isEmpty: Bool { _count == 0 } - - public typealias Index = Int - - @_alwaysEmitIntoClient - public var indices: Range { - Range(uncheckedBounds: (0, _count)) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: ~Copyable { - - /// Accesses the element at the specified position in the `Span`. - /// - /// - Parameter position: The offset of the element to access. `position` - /// must be greater or equal to zero, and less than `count`. - /// - /// - Complexity: O(1) - @_alwaysEmitIntoClient - public subscript(_ position: Index) -> Element { - _read { - precondition(indices.contains(position), "index out of bounds") - yield self[unchecked: position] - } - @_lifetime(self: copy self) - _modify { - precondition(indices.contains(position), "index out of bounds") - yield &self[unchecked: position] - } - } - - /// Accesses the element at the specified position in the `Span`. - /// - /// This subscript does not validate `position`; this is an unsafe operation. - /// - /// - Parameter position: The offset of the element to access. `position` - /// must be greater or equal to zero, and less than `count`. - /// - /// - Complexity: O(1) - @_alwaysEmitIntoClient - public subscript(unchecked position: Index) -> Element { - unsafeAddress { - UnsafePointer(_unsafeAddressOfElement(unchecked: position)) - } - @_lifetime(self: copy self) - unsafeMutableAddress { - _unsafeAddressOfElement(unchecked: position) - } - } - - @unsafe - @_alwaysEmitIntoClient - internal func _unsafeAddressOfElement( - unchecked position: Index - ) -> UnsafeMutablePointer { - let elementOffset = position &* MemoryLayout.stride - let address = _start().advanced(by: elementOffset) - return address.assumingMemoryBound(to: Element.self) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: ~Copyable { - - @_lifetime(self: copy self) - public mutating func swapAt(_ i: Index, _ j: Index) { - precondition(indices.contains(Index(i))) - precondition(indices.contains(Index(j))) - swapAt(unchecked: i, unchecked: j) - } - - @_lifetime(self: copy self) - public mutating func swapAt(unchecked i: Index, unchecked j: Index) { - let pi = _unsafeAddressOfElement(unchecked: i) - let pj = _unsafeAddressOfElement(unchecked: j) - let temporary = pi.move() - pi.initialize(to: pj.move()) - pj.initialize(to: consume temporary) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: BitwiseCopyable { - - /// Accesses the element at the specified position in the `Span`. - /// - /// - Parameter position: The offset of the element to access. `position` - /// must be greater or equal to zero, and less than `count`. - /// - /// - Complexity: O(1) - @_alwaysEmitIntoClient - public subscript(_ position: Index) -> Element { - get { - precondition(indices.contains(position)) - return self[unchecked: position] - } - @_lifetime(self: copy self) - set { - precondition(indices.contains(position)) - self[unchecked: position] = newValue - } - } - - /// Accesses the element at the specified position in the `Span`. - /// - /// This subscript does not validate `position`; this is an unsafe operation. - /// - /// - Parameter position: The offset of the element to access. `position` - /// must be greater or equal to zero, and less than `count`. - /// - /// - Complexity: O(1) - @_alwaysEmitIntoClient - public subscript(unchecked position: Index) -> Element { - get { - let offset = position&*MemoryLayout.stride - return _start().loadUnaligned(fromByteOffset: offset, as: Element.self) - } - @_lifetime(self: copy self) - set { - let offset = position&*MemoryLayout.stride - _start().storeBytes(of: newValue, toByteOffset: offset, as: Element.self) - } - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: ~Copyable { - - //FIXME: mark closure parameter as non-escaping - @_alwaysEmitIntoClient - public func withUnsafeBufferPointer( - _ body: (_ buffer: UnsafeBufferPointer) throws(E) -> Result - ) throws(E) -> Result { - try Span(_unsafeMutableSpan: self).withUnsafeBufferPointer(body) - } - - //FIXME: mark closure parameter as non-escaping - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func withUnsafeMutableBufferPointer( - _ body: (UnsafeMutableBufferPointer) throws(E) -> Result - ) throws(E) -> Result { - guard let pointer = _pointer, count > 0 else { - return try body(.init(start: nil, count: 0)) - } - // bind memory by hand to sidestep alignment concerns - let binding = Builtin.bindMemory( - pointer._rawValue, count._builtinWordValue, Element.self - ) - defer { Builtin.rebindMemory(pointer._rawValue, binding) } - return try body(.init(start: .init(pointer._rawValue), count: count)) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan where Element: BitwiseCopyable { - - //FIXME: mark closure parameter as non-escaping - @_alwaysEmitIntoClient - public func withUnsafeBytes( - _ body: (_ buffer: UnsafeRawBufferPointer) throws(E) -> Result - ) throws(E) -> Result { - try RawSpan(_unsafeMutableSpan: self).withUnsafeBytes(body) - } - - //FIXME: mark closure parameter as non-escaping - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func withUnsafeMutableBytes( - _ body: (_ buffer: UnsafeMutableRawBufferPointer) throws(E) -> Result - ) throws(E) -> Result { - let bytes = UnsafeMutableRawBufferPointer( - start: (_count == 0) ? nil : _start(), - count: _count &* MemoryLayout.stride - ) - return try body(bytes) - } -} - -//MARK: bulk-update functions -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension MutableSpan { - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update(repeating repeatedValue: Element) { - _start().withMemoryRebound(to: Element.self, capacity: count) { - $0.update(repeating: repeatedValue, count: count) - } - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - from source: S - ) -> (unwritten: S.Iterator, index: Index) where S.Element == Element { - var iterator = source.makeIterator() - let index = update(from: &iterator) - return (iterator, index) - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - from elements: inout some IteratorProtocol - ) -> Index { - var index = 0 - while index < _count { - guard let element = elements.next() else { break } - self[unchecked: index] = element - index &+= 1 - } - return index - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - fromContentsOf source: some Collection - ) -> Index { - let updated = source.withContiguousStorageIfAvailable { - self.update(fromContentsOf: Span(_unsafeElements: $0)) - } - if let updated { - return updated - } - - var iterator = source.makeIterator() - let index = update(from: &iterator) - precondition( - iterator.next() == nil, - "destination buffer view cannot contain every element from source." - ) - return index - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update(fromContentsOf source: Span) -> Index { - guard !source.isEmpty else { return 0 } - precondition( - source.count <= self.count, - "destination span cannot contain every element from source." - ) - _start().withMemoryRebound(to: Element.self, capacity: source.count) { dest in - source.withUnsafeBufferPointer { - dest.update(from: $0.baseAddress!, count: $0.count) - } - } - return source.count - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - fromContentsOf source: borrowing MutableSpan - ) -> Index { - update(fromContentsOf: source.storage) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) - -extension MutableSpan where Element: BitwiseCopyable { - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - repeating repeatedValue: Element - ) where Element: BitwiseCopyable { - guard count > 0 else { return } - // rebind _start manually in order to avoid assumptions about alignment. - let rp = _start()._rawValue - let binding = Builtin.bindMemory(rp, count._builtinWordValue, Element.self) - UnsafeMutablePointer(rp).update(repeating: repeatedValue, count: count) - Builtin.rebindMemory(rp, binding) - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - from source: S - ) -> (unwritten: S.Iterator, index: Index) - where S.Element == Element, Element: BitwiseCopyable { - var iterator = source.makeIterator() - let index = update(from: &iterator) - return (iterator, index) - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - from elements: inout some IteratorProtocol - ) -> Index { - var index = 0 - while index < _count { - guard let element = elements.next() else { break } - self[unchecked: index] = element - index &+= 1 - } - return index - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - fromContentsOf source: some Collection - ) -> Index where Element: BitwiseCopyable { - let updated = source.withContiguousStorageIfAvailable { - self.update(fromContentsOf: Span(_unsafeElements: $0)) - } - if let updated { - return updated - } - - var iterator = source.makeIterator() - let index = update(from: &iterator) - precondition( - iterator.next() == nil, - "destination buffer view cannot contain every element from source." - ) - return index - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - fromContentsOf source: Span - ) -> Index where Element: BitwiseCopyable { - guard !source.isEmpty else { return 0 } - precondition( - source.count <= self.count, - "destination span cannot contain every element from source." - ) - source.withUnsafeBufferPointer { - _start().copyMemory( - from: $0.baseAddress!, byteCount: $0.count&*MemoryLayout.stride - ) - } - return source.count - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func update( - fromContentsOf source: borrowing MutableSpan - ) -> Index where Element: BitwiseCopyable { - update(fromContentsOf: source.storage) - } -} - -@frozen -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -public struct OutputSpan: ~Copyable, ~Escapable { - @usableFromInline let _pointer: UnsafeMutableRawPointer? - - public let capacity: Int - - @usableFromInline - var _initialized: Int = 0 - - @_alwaysEmitIntoClient - @usableFromInline @inline(__always) - var _start: UnsafeMutableRawPointer { _pointer.unsafelyUnwrapped } - - @_alwaysEmitIntoClient - public var available: Int { capacity &- _initialized } - - @_alwaysEmitIntoClient - public var count: Int { _initialized } - - @_alwaysEmitIntoClient - public var isEmpty: Bool { _initialized == 0 } - - deinit { - if _initialized > 0 { - _start.withMemoryRebound(to: Element.self, capacity: _initialized) { - [ workaround = _initialized ] in - _ = $0.deinitialize(count: workaround) - } - } - } - - @usableFromInline @inline(__always) - @_lifetime(borrow start) - init( - _unchecked start: UnsafeMutableRawPointer?, - capacity: Int, - initialized: Int - ) { - _pointer = start - self.capacity = capacity - _initialized = initialized - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -@available(*, unavailable) -extension OutputSpan: Sendable {} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan where Element: ~Copyable { - @unsafe - @_alwaysEmitIntoClient - @_lifetime(borrow buffer) - public init( - _uncheckedBuffer buffer: UnsafeMutableBufferPointer, - initializedCount: Int - ) { - unsafe _pointer = .init(buffer.baseAddress) - capacity = buffer.count - _initialized = initializedCount - } - - /// Unsafely create an OutputSpan over partly-initialized memory. - /// - /// The memory in `buffer` must remain valid throughout the lifetime - /// of the newly-created `OutputSpan`. Its prefix must contain - /// `initializedCount` initialized instances, followed by uninitialized - /// memory. The default value of `initializedCount` is 0, representing - /// the common case of a completely uninitialized `buffer`. - /// - /// - Parameters: - /// - buffer: an `UnsafeMutableBufferPointer` to be initialized - /// - initializedCount: the number of initialized elements - /// at the beginning of `buffer`. - @unsafe - @_alwaysEmitIntoClient - @_lifetime(borrow buffer) - public init( - buffer: UnsafeMutableBufferPointer, - initializedCount: Int - ) { - precondition(buffer._isWellAligned(), "Misaligned OutputSpan") - if let baseAddress = buffer.baseAddress { - precondition( - unsafe baseAddress.advanced(by: buffer.count) >= baseAddress, - "Buffer must not wrap around the address space" - ) - } - precondition( - 0 <= initializedCount && initializedCount <= buffer.count, - "OutputSpan count is not within capacity" - ) - unsafe self.init( - _uncheckedBuffer: buffer, initializedCount: initializedCount - ) - } -} - -extension UnsafePointer where Pointee: ~Copyable { - @safe - @_alwaysEmitIntoClient - public func _isWellAligned() -> Bool { - (Int(bitPattern: self) & (MemoryLayout.alignment &- 1)) == 0 - } -} - -extension UnsafeMutablePointer where Pointee: ~Copyable { - @safe - @_alwaysEmitIntoClient - public func _isWellAligned() -> Bool { - (Int(bitPattern: self) & (MemoryLayout.alignment &- 1)) == 0 - } -} - -extension UnsafeBufferPointer where Element: ~Copyable { - @safe - @_alwaysEmitIntoClient - public func _isWellAligned() -> Bool { - guard let p = baseAddress else { return true } - return p._isWellAligned() - } -} - -extension UnsafeMutableBufferPointer where Element: ~Copyable { - @inlinable - public func _isWellAligned() -> Bool { - guard let p = baseAddress else { return true } - return p._isWellAligned() - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan where Element: ~Copyable { - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func append(_ value: consuming Element) { - precondition(_initialized < capacity, "Output buffer overflow") - let p = _start.advanced(by: _initialized&*MemoryLayout.stride) - p.initializeMemory(as: Element.self, to: value) - _initialized &+= 1 - } - - @_alwaysEmitIntoClient - public mutating func deinitializeLastElement() -> Element? { - guard _initialized > 0 else { return nil } - _initialized &-= 1 - let p = _start.advanced(by: _initialized&*MemoryLayout.stride) - return p.withMemoryRebound(to: Element.self, capacity: 1, { $0.move() }) - } - - @_alwaysEmitIntoClient - public mutating func deinitialize() { - _ = _start.withMemoryRebound(to: Element.self, capacity: _initialized) { - $0.deinitialize(count: _initialized) - } - _initialized = 0 - } -} - -//MARK: bulk-update functions -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan { - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func append(repeating repeatedValue: Element, count: Int) { - let available = capacity &- _initialized - precondition( - count <= available, - "destination span cannot contain number of elements requested." - ) - let offset = _initialized&*MemoryLayout.stride - let p = _start.advanced(by: offset) - p.withMemoryRebound(to: Element.self, capacity: count) { - $0.initialize(repeating: repeatedValue, count: count) - } - _initialized &+= count - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func append( - from elements: S - ) -> S.Iterator where S: Sequence, S.Element == Element { - var iterator = elements.makeIterator() - append(from: &iterator) - return iterator - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func append( - from elements: inout some IteratorProtocol - ) { - while _initialized < capacity { - guard let element = elements.next() else { break } - let p = _start.advanced(by: _initialized&*MemoryLayout.stride) - p.initializeMemory(as: Element.self, to: element) - _initialized &+= 1 - } - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func append( - fromContentsOf source: some Collection - ) { - let void: Void? = source.withContiguousStorageIfAvailable { - append(fromContentsOf: Span(_unsafeElements: $0)) - } - if void != nil { - return - } - - let available = capacity &- _initialized - let tail = _start.advanced(by: _initialized&*MemoryLayout.stride) - var (iterator, copied) = - tail.withMemoryRebound(to: Element.self, capacity: available) { - let suffix = UnsafeMutableBufferPointer(start: $0, count: available) - return source._copyContents(initializing: suffix) - } - precondition( - iterator.next() == nil, - "destination span cannot contain every element from source." - ) - assert(_initialized + copied <= capacity) // invariant check - _initialized &+= copied - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func append( - fromContentsOf source: Span - ) { - guard !source.isEmpty else { return } - precondition( - source.count <= available, - "destination span cannot contain every element from source." - ) - let tail = _start.advanced(by: _initialized&*MemoryLayout.stride) - _ = source.withUnsafeBufferPointer { - tail.initializeMemory( - as: Element.self, from: $0.baseAddress!, count: $0.count - ) - } - _initialized += source.count - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func append(fromContentsOf source: borrowing MutableSpan) { - source.withUnsafeBufferPointer { append(fromContentsOf: $0) } - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan where Element: ~Copyable { - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func moveAppend( - fromContentsOf source: consuming Self - ) { - guard !source.isEmpty else { return } - precondition( - source.count <= available, - "buffer cannot contain every element from source." - ) - let buffer = source.relinquishBorrowedMemory() - // we must now deinitialize the returned UMBP - let tail = _start.advanced(by: _initialized&*MemoryLayout.stride) - tail.moveInitializeMemory( - as: Element.self, from: buffer.baseAddress!, count: buffer.count - ) - _initialized &+= buffer.count - } - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func moveAppend( - fromContentsOf source: UnsafeMutableBufferPointer - ) { - let source = OutputSpan(buffer: source, initializedCount: source.count) - moveAppend(fromContentsOf: source) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan { - - @_alwaysEmitIntoClient - @_lifetime(self: copy self) - public mutating func moveAppend( - fromContentsOf source: Slice> - ) { - moveAppend(fromContentsOf: UnsafeMutableBufferPointer(rebasing: source)) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan where Element: BitwiseCopyable { - -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan where Element: ~Copyable { - - @_alwaysEmitIntoClient - public var span: Span { - @_lifetime(borrow self) - borrowing get { - let pointer = _pointer?.assumingMemoryBound(to: Element.self) - let buffer = UnsafeBufferPointer(start: pointer, count: _initialized) - let span = Span(_unsafeElements: buffer) - return _overrideLifetime(span, borrowing: self) - } - } - - @_alwaysEmitIntoClient - public var mutableSpan: MutableSpan { - @_lifetime(&self) - mutating get { // the accessor must provide a mutable projection - let pointer = _pointer?.assumingMemoryBound(to: Element.self) - let buffer = UnsafeMutableBufferPointer(start: pointer, count: _initialized) - let span = MutableSpan(_unsafeElements: buffer) - return _overrideLifetime(span, mutating: &self) - } - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan where Element: ~Copyable { - - @_alwaysEmitIntoClient - public consuming func relinquishBorrowedMemory() -> UnsafeMutableBufferPointer { - let (start, count) = (self._pointer, self._initialized) - discard self - let typed = start?.bindMemory(to: Element.self, capacity: count) - return .init(start: typed, count: count) - } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension OutputSpan where Element: BitwiseCopyable { - - @_alwaysEmitIntoClient - public consuming func relinquishBorrowedBytes() -> UnsafeMutableRawBufferPointer { - let (start, count) = (self._pointer, self._initialized) - discard self - return .init(start: start, count: count&*MemoryLayout.stride) - } -} - -private let immortalThing = "" - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension Span { - -// @available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -// public static var empty: Span { -// @_lifetime(immortal) -// get { -// let nilBasedBuffer = UnsafeBufferPointer(start: nil, count: 0) -// let span = Span(_unsafeElements: nilBasedBuffer) -// return _overrideLifetime(span, to: immortalThing) -// } -// } -// -// @available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -// @_lifetime(immortal) -// public init() { -// let nilBasedBuffer = UnsafeBufferPointer(start: nil, count: 0) -// let span = Span(_unsafeElements: nilBasedBuffer) -// self = _overrideLifetime(span, to: immortalThing) -// } -} - -@available(macOS 9999, iOS 9999, tvOS 9999, watchOS 9999, visionOS 9999, *) -extension Span where Element: Equatable { - - /// Returns a Boolean value indicating whether this and another span - /// contain equal elements in the same order. - /// - /// - Parameters: - /// - other: A span to compare to this one. - /// - Returns: `true` if this sequence and `other` contain equivalent items, - /// using `areEquivalent` as the equivalence test; otherwise, `false.` - /// - /// - Complexity: O(*m*), where *m* is the lesser of the length of the - /// sequence and the length of `other`. - @_alwaysEmitIntoClient - public func _elementsEqual(_ other: Self) -> Bool { - guard count == other.count else { return false } - if count == 0 { return true } - - //FIXME: This could be short-cut - // with a layout constraint where stride equals size, - // as long as there is at most 1 unused bit pattern. - // if Element is BitwiseEquatable { - // return _swift_stdlib_memcmp(lhs.baseAddress, rhs.baseAddress, count) == 0 - // } - for o in 0..) -> Bool { - let equal = other.withContiguousStorageIfAvailable { - _elementsEqual(Span(_unsafeElements: $0)) - } - if let equal { return equal } - - guard count == other.count else { return false } - if count == 0 { return true } - - return _elementsEqual(AnySequence(other)) - } - - /// Returns a Boolean value indicating whether this span and a Sequence - /// contain equal elements in the same order. - /// - /// - Parameters: - /// - other: A Sequence to compare to this span. - /// - Returns: `true` if this sequence and `other` contain equivalent items, - /// using `areEquivalent` as the equivalence test; otherwise, `false.` - /// - /// - Complexity: O(*m*), where *m* is the lesser of the length of the - /// sequence and the length of `other`. - @_alwaysEmitIntoClient - public func _elementsEqual(_ other: some Sequence) -> Bool { - var offset = 0 - for otherElement in other { - if offset >= count { return false } - if self[unchecked: offset] != otherElement { return false } - offset += 1 - } - return offset == count - } -} - diff --git a/test/SILOptimizer/mutable_span_bounds_check_tests.swift b/test/SILOptimizer/mutable_span_bounds_check_tests.swift index 5b6544b2b69da..d8b234699d847 100644 --- a/test/SILOptimizer/mutable_span_bounds_check_tests.swift +++ b/test/SILOptimizer/mutable_span_bounds_check_tests.swift @@ -1,6 +1,5 @@ // RUN: %empty-directory(%t) -// RUN: %target-swift-frontend -emit-module-path %t/SpanExtras.swiftmodule %S/Inputs/SpanExtras.swift -enable-builtin-module -enable-experimental-feature Lifetimes -O -// RUN: %target-swift-frontend -I %t -O -emit-sil %s -enable-experimental-feature Lifetimes -disable-availability-checking | %FileCheck %s --check-prefix=CHECK-SIL +// RUN: %target-swift-frontend -I %t -O -emit-sil %s -enable-experimental-feature Lifetimes -disable-availability-checking | %FileCheck %s --check-prefix=CHECK-SIL // RUN: %target-swift-frontend -I %t -O -emit-ir %s -enable-experimental-feature Lifetimes -disable-availability-checking | %FileCheck %s --check-prefix=CHECK-IR // REQUIRES: swift_in_compiler @@ -8,40 +7,75 @@ // REQUIRES: swift_stdlib_no_asserts, optimized_stdlib -import SpanExtras +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0a1_B11_sum_w_trapySis11MutableSpanVySiGF : +// CHECK-SIL: bb3({{.*}}): +// CHECK-SIL-NOT: cond_fail {{.*}}, "index out of bounds" +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0a1_B11_sum_w_trapySis11MutableSpanVySiGF' +public func mutable_span_sum_w_trap(_ ms: borrowing MutableSpan) -> Int { + var sum = 0 + for i in ms.indices { + sum += ms[i] + } + return sum +} + +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0a1_B12_sum_wo_trapySis11MutableSpanVySiGF : +// CHECK-SIL: bb3({{.*}}): +// CHECK-SIL-NOT: cond_fail {{.*}}, "index out of bounds" +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0a1_B12_sum_wo_trapySis11MutableSpanVySiGF' +public func mutable_span_sum_wo_trap(_ ms: borrowing MutableSpan) -> Int { + var sum = 0 + for i in ms.indices { + sum &+= ms[i] + } + return sum +} -// Bounds check should be eliminated -// SIL removes lower bounds check -// LLVM removes upper bounds check and vectorizes the loop +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0a1_B25_sum_w_trap_unknown_boundySis11MutableSpanVySiG_SitF : +// CHECK-SIL: bb3({{.*}}): +// CHECK-SIL-NOT: cond_fail {{.*}}, "index out of bounds" +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0a1_B25_sum_w_trap_unknown_boundySis11MutableSpanVySiG_SitF' +public func mutable_span_sum_w_trap_unknown_bound(_ ms: borrowing MutableSpan, _ n: Int) -> Int { + var sum = 0 + for i in 0...n { + sum += ms[i] + } + return sum +} -// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0B10_zero_inityy10SpanExtras07MutableH0VySiGzF : +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0a1_B26_sum_wo_trap_unknown_boundySis11MutableSpanVySiG_SitF : // CHECK-SIL: bb3({{.*}}): -// CHECK-SIL: cond_fail {{.*}}, "precondition failure" -// CHECK-SIL-NOT: cond_fail {{.*}}, "precondition failure" +// CHECK-SIL-NOT: cond_fail {{.*}}, "index out of bounds" +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0a1_B26_sum_wo_trap_unknown_boundySis11MutableSpanVySiG_SitF' +public func mutable_span_sum_wo_trap_unknown_bound(_ ms: borrowing MutableSpan, _ n: Int) -> Int { + var sum = 0 + for i in 0...n { + sum &+= ms[i] + } + return sum +} + +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0B10_zero_inityys11MutableSpanVySiGzF : +// CHECK-SIL: bb3({{.*}}): +// CHECK-SIL-NOT: cond_fail {{.*}}, "index out of bounds" // CHECK-SIL: cond_br -// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0B10_zero_inityy10SpanExtras07MutableH0VySiGzF' +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0B10_zero_inityys11MutableSpanVySiGzF' -// CHECK-IR: define {{.*}} void @"$s31mutable_span_bounds_check_tests0B10_zero_inityy10SpanExtras07MutableH0VySiGzF" -// CHECK-IR: vector.body -// CHECK-IR: store <{{.*}}> zeroinitializer, +// CHECK-IR: define {{.*}} void @"$s31mutable_span_bounds_check_tests0B10_zero_inityys11MutableSpanVySiGzF" +// CHECK-IR: call void @llvm.memset public func span_zero_init(_ output: inout MutableSpan) { for i in output.indices { output[i] = 0 } } -// Bounds check should be eliminated -// SIL removes lower bounds check -// LLVM removes upper bounds check and vectorizes the loop - -// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0B14_copy_elemwiseyy10SpanExtras07MutableH0VySiGz_s0H0VySiGtF : +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0B14_copy_elemwiseyys11MutableSpanVySiGz_s0I0VySiGtF : // CHECK-SIL: bb3({{.*}}): -// CHECK-SIL: cond_fail {{.*}}, "precondition failure" -// CHECK-SIL-NOT: cond_fail {{.*}}, "precondition failure" +// CHECK-SIL-NOT: cond_fail {{.*}}, "index out of bounds" // CHECK-SIL: cond_br -// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0B14_copy_elemwiseyy10SpanExtras07MutableH0VySiGz_s0H0VySiGtF' +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0B14_copy_elemwiseyys11MutableSpanVySiGz_s0I0VySiGtF' -// CHECK-IR: define {{.*}} void @"$s31mutable_span_bounds_check_tests0B14_copy_elemwiseyy10SpanExtras07MutableH0VySiGz_s0H0VySiGtF" +// CHECK-IR: define {{.*}} void @"$s31mutable_span_bounds_check_tests0B14_copy_elemwiseyys11MutableSpanVySiGz_s0I0VySiGtF" // CHECK-IR: vector.body // CHECK-IR: store <{{.*}}> @_lifetime(output: copy output, copy input) @@ -52,18 +86,13 @@ public func span_copy_elemwise(_ output: inout MutableSpan, _ input: Span @_lifetime(output: copy output, copy input) @@ -73,16 +102,13 @@ public func span_append_elemwise(_ output: inout OutputSpan, _ input: Span< } } -// Bounds check should be eliminated - -// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0B12_sum_wo_trapyy10SpanExtras07MutableI0VySiGz_s0I0VySiGAItF : +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0B12_sum_wo_trapyys11MutableSpanVySiGz_s0J0VySiGAHtF : // CHECK-SIL: bb3({{.*}}): -// CHECK-SIL: cond_fail {{.*}}, "precondition failure" -// CHECK-SIL-NOT: cond_fail {{.*}}, "precondition failure" +// CHECK-SIL-NOT: cond_fail {{.*}}, "index out of bounds" // CHECK-SIL: cond_br -// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0B12_sum_wo_trapyy10SpanExtras07MutableI0VySiGz_s0I0VySiGAItF' +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0B12_sum_wo_trapyys11MutableSpanVySiGz_s0J0VySiGAHtF' -// CHECK-IR: define {{.*}} void @"$s31mutable_span_bounds_check_tests0B12_sum_wo_trapyy10SpanExtras07MutableI0VySiGz_s0I0VySiGAItF" +// CHECK-IR: define {{.*}} void @"$s31mutable_span_bounds_check_tests0B12_sum_wo_trapyys11MutableSpanVySiGz_s0J0VySiGAHtF" // CHECK-IR: vector.body // CHECK-IR: store <{{.*}}> @_lifetime(output: copy output, copy input1, copy input2) @@ -94,12 +120,11 @@ public func span_sum_wo_trap(_ output: inout MutableSpan, _ input1: Span, _ input1: Span, _ input2: Span) { precondition(input1.count == input2.count) @@ -109,12 +134,10 @@ public func span_sum_with_trap(_ output: inout MutableSpan, _ input1: Span< } } -// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0B12_bubble_sortyy10SpanExtras07MutableH0VySiGzF : -// CHECK-SIL: bb11({{.*}}): -// CHECK-SIL: cond_fail {{.*}}, "precondition failure" -// CHECK-SIL: cond_fail {{.*}}, "precondition failure" -// CHECK-SIL: cond_br -// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0B12_bubble_sortyy10SpanExtras07MutableH0VySiGzF' +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests0B12_bubble_sortyys11MutableSpanVySiGzF : +// CHECK-SIL: bb12{{.*}}: +// TODO-CHECK-SIL-NOT: cond_fail {{.*}}, "index out of bounds" +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests0B12_bubble_sortyys11MutableSpanVySiGzF' public func span_bubble_sort(_ span: inout MutableSpan) { if span.count <= 1 { return @@ -130,12 +153,12 @@ public func span_bubble_sort(_ span: inout MutableSpan) { } } -// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests6sortedySb10SpanExtras07MutableG0VySiGF : +// CHECK-SIL-LABEL: sil @$s31mutable_span_bounds_check_tests6sortedySbs11MutableSpanVySiGF : // CHECK-SIL: bb4: -// CHECK-SIL: cond_fail {{.*}}, "precondition failure" -// CHECK-SIL: cond_fail {{.*}}, "precondition failure" +// CHECK-SIL: cond_fail {{.*}}, "index out of bounds" +// CHECK-SIL: cond_fail {{.*}}, "index out of bounds" // CHECK-SIL: cond_br -// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests6sortedySb10SpanExtras07MutableG0VySiGF' +// CHECK-SIL-LABEL: } // end sil function '$s31mutable_span_bounds_check_tests6sortedySbs11MutableSpanVySiGF' public func sorted(_ span: borrowing MutableSpan) -> Bool { if span.count <= 1 { return true