diff --git a/Sources/XCLogParser/activityparser/ActivityParser.swift b/Sources/XCLogParser/activityparser/ActivityParser.swift index 07d8f27..8aec978 100644 --- a/Sources/XCLogParser/activityparser/ActivityParser.swift +++ b/Sources/XCLogParser/activityparser/ActivityParser.swift @@ -380,12 +380,40 @@ public class ActivityParser { } if className == "IDEFoundation.\(String(describing: IDEActivityLogSectionAttachment.self))" { - let jsonType = IDEActivityLogSectionAttachment.BuildOperationTaskMetrics.self - return try IDEActivityLogSectionAttachment(identifier: try parseAsString(token: iterator.next()), - majorVersion: try parseAsInt(token: iterator.next()), - minorVersion: try parseAsInt(token: iterator.next()), - metrics: try parseAsJson(token: iterator.next(), - type: jsonType)) + let identifier = try parseAsString(token: iterator.next()) + switch identifier.components(separatedBy: "ActivityLogSectionAttachment.").last { + case .some("TaskMetrics"): + let jsonType = IDEActivityLogSectionAttachment.BuildOperationTaskMetrics.self + return try IDEActivityLogSectionAttachment(identifier: identifier, + majorVersion: try parseAsInt(token: iterator.next()), + minorVersion: try parseAsInt(token: iterator.next()), + metrics: try parseAsJson(token: iterator.next(), + type: jsonType), + buildOperationMetrics: nil, + backtrace: nil) + case .some("TaskBacktrace"): + let jsonType = IDEActivityLogSectionAttachment.BuildOperationTaskBacktrace.self + return try IDEActivityLogSectionAttachment(identifier: identifier, + majorVersion: try parseAsInt(token: iterator.next()), + minorVersion: try parseAsInt(token: iterator.next()), + metrics: nil, + buildOperationMetrics: nil, + backtrace: try parseAsJson(token: iterator.next(), + type: jsonType)) + case .some("BuildOperationMetrics"): + let jsonType = IDEActivityLogSectionAttachment.BuildOperationMetrics.self + return try IDEActivityLogSectionAttachment(identifier: identifier, + majorVersion: try parseAsInt(token: iterator.next()), + minorVersion: try parseAsInt(token: iterator.next()), + metrics: nil, + buildOperationMetrics: try parseAsJson( + token: iterator.next(), + type: jsonType + ), + backtrace: nil) + default: + throw XCLogParserError.parseError("Unexpected attachment identifier \(identifier)") + } } throw XCLogParserError.parseError("Unexpected className found parsing IDEConsoleItem \(className)") } diff --git a/Sources/XCLogParser/activityparser/IDEActivityModel.swift b/Sources/XCLogParser/activityparser/IDEActivityModel.swift index adeb83f..312a04d 100644 --- a/Sources/XCLogParser/activityparser/IDEActivityModel.swift +++ b/Sources/XCLogParser/activityparser/IDEActivityModel.swift @@ -658,16 +658,137 @@ public class IDEActivityLogSectionAttachment: Encodable { public let majorVersion: UInt64 public let minorVersion: UInt64 public let metrics: BuildOperationTaskMetrics? + public let buildOperationMetrics: BuildOperationMetrics? + public let backtrace: BuildOperationTaskBacktrace? public init( identifier: String, majorVersion: UInt64, minorVersion: UInt64, - metrics: BuildOperationTaskMetrics? + metrics: BuildOperationTaskMetrics?, + buildOperationMetrics: BuildOperationMetrics?, + backtrace: BuildOperationTaskBacktrace? ) throws { self.identifier = identifier self.majorVersion = majorVersion self.minorVersion = minorVersion self.metrics = metrics + self.buildOperationMetrics = buildOperationMetrics + self.backtrace = backtrace + } + + public struct BuildOperationTaskBacktrace: Codable { + public let frames: [BuildOperationTaskBacktraceFrame] + + public init(from decoder: any Decoder) throws { + let container = try decoder.singleValueContainer() + self.frames = try container.decode([BuildOperationTaskBacktraceFrame].self) + } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(frames) + } + } + + public struct BuildOperationTaskBacktraceFrame: Codable { + public let category: BuildOperationTaskBacktraceCategory + public let description: String + } + + public enum BuildOperationTaskBacktraceCategory: String, Codable { + case ruleHadInvalidValue + case ruleSignatureChanged + case ruleNeverBuilt + case ruleInputRebuilt + case ruleForced + case dynamicTaskRegistration + case dynamicTaskRequest + case none + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: BuildOperationTaskBacktraceCategoryCodingKeys.self) + + if container.contains(.ruleHadInvalidValue) { + self = .ruleHadInvalidValue + } else if container.contains(.ruleSignatureChanged) { + self = .ruleSignatureChanged + } else if container.contains(.ruleNeverBuilt) { + self = .ruleNeverBuilt + } else if container.contains(.ruleInputRebuilt) { + self = .ruleInputRebuilt + } else if container.contains(.ruleForced) { + self = .ruleForced + } else if container.contains(.dynamicTaskRegistration) { + self = .dynamicTaskRegistration + } else if container.contains(.dynamicTaskRequest) { + self = .dynamicTaskRequest + } else if container.contains(.none) { + self = .none + } else { + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown task backtrace category" + ) + ) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: BuildOperationTaskBacktraceCategoryCodingKeys.self) + switch self { + case .ruleHadInvalidValue: + try container.encode(EmptyObject(), forKey: .ruleHadInvalidValue) + case .ruleSignatureChanged: + try container.encode(EmptyObject(), forKey: .ruleSignatureChanged) + case .ruleNeverBuilt: + try container.encode(EmptyObject(), forKey: .ruleNeverBuilt) + case .ruleInputRebuilt: + try container.encode(EmptyObject(), forKey: .ruleInputRebuilt) + case .ruleForced: + try container.encode(EmptyObject(), forKey: .ruleForced) + case .dynamicTaskRegistration: + try container.encode(EmptyObject(), forKey: .dynamicTaskRegistration) + case .dynamicTaskRequest: + try container.encode(EmptyObject(), forKey: .dynamicTaskRequest) + case .none: + try container.encode(EmptyObject(), forKey: .none) + } + } + } + + private enum BuildOperationTaskBacktraceCategoryCodingKeys: String, CodingKey { + case ruleHadInvalidValue + case ruleSignatureChanged + case ruleNeverBuilt + case ruleInputRebuilt + case ruleForced + case dynamicTaskRegistration + case dynamicTaskRequest + case none + } + + private struct EmptyObject: Codable { + // Empty struct for objects with no properties + } + + public struct BuildOperationMetrics: Codable { + public let clangCacheHits: Int + public let clangCacheMisses: Int + public let swiftCacheHits: Int + public let swiftCacheMisses: Int + + public init( + clangCacheHits: Int, + clangCacheMisses: Int, + swiftCacheHits: Int, + swiftCacheMisses: Int + ) { + self.clangCacheHits = clangCacheHits + self.clangCacheMisses = clangCacheMisses + self.swiftCacheHits = swiftCacheHits + self.swiftCacheMisses = swiftCacheMisses + } } } diff --git a/Sources/XCLogParser/commands/Version.swift b/Sources/XCLogParser/commands/Version.swift index 84c80cb..f4dec17 100644 --- a/Sources/XCLogParser/commands/Version.swift +++ b/Sources/XCLogParser/commands/Version.swift @@ -21,6 +21,6 @@ import Foundation public struct Version { - public static let current = "0.2.39" + public static let current = "0.2.43" } diff --git a/Sources/XCLogParser/lexer/Index+Offset.swift b/Sources/XCLogParser/lexer/Index+Offset.swift new file mode 100644 index 0000000..41daca8 --- /dev/null +++ b/Sources/XCLogParser/lexer/Index+Offset.swift @@ -0,0 +1,30 @@ +// Copyright (c) 2019 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +extension String.Index { + init(compilerSafeOffset offset: Int, in string: String) { +#if swift(>=5.0) + self = String.Index(utf16Offset: offset, in: string) +#else + self = String.Index(encodedOffset: offset) +#endif + } +} diff --git a/Sources/XCLogParser/lexer/Lexer.swift b/Sources/XCLogParser/lexer/Lexer.swift index 03ad44b..cbe5225 100644 --- a/Sources/XCLogParser/lexer/Lexer.swift +++ b/Sources/XCLogParser/lexer/Lexer.swift @@ -23,7 +23,7 @@ public final class Lexer { static let SLFHeader = "SLF" - let typeDelimiters: CharacterSet + let typeDelimiters: Set let filePath: String var classNames = [String]() var userDirToRedact: String? { @@ -38,7 +38,7 @@ public final class Lexer { public init(filePath: String) { self.filePath = filePath - self.typeDelimiters = CharacterSet(charactersIn: TokenType.all()) + self.typeDelimiters = Set(TokenType.all()) self.redactor = LexRedactor() } @@ -53,15 +53,18 @@ public final class Lexer { redacted: Bool, withoutBuildSpecificInformation: Bool) throws -> [Token] { let scanner = Scanner(string: contents) + guard scanSLFHeader(scanner: scanner) else { throw XCLogParserError.invalidLogHeader(filePath) } + var tokens = [Token]() while !scanner.isAtEnd { + guard let logTokens = scanSLFType(scanner: scanner, redacted: redacted, withoutBuildSpecificInformation: withoutBuildSpecificInformation), - logTokens.isEmpty == false else { + logTokens.isEmpty == false else { print(tokens) throw XCLogParserError.invalidLine(scanner.approximateLine) } @@ -71,20 +74,15 @@ public final class Lexer { } private func scanSLFHeader(scanner: Scanner) -> Bool { - #if os(Linux) - var format: String? - #else - var format: NSString? - #endif - return scanner.scanString(Lexer.SLFHeader, into: &format) + return scanner.scan(string: Lexer.SLFHeader) } - private func scanSLFType(scanner: Scanner, redacted: Bool, withoutBuildSpecificInformation: Bool) -> [Token]? { + private func scanSLFType(scanner: Scanner, + redacted: Bool, + withoutBuildSpecificInformation: Bool) -> [Token]? { + let payload = self.scanPayload(scanner: scanner) - guard let payload = scanPayload(scanner: scanner) else { - return nil - } - guard let tokenTypes = scanTypeDelimiter(scanner: scanner), tokenTypes.count > 0 else { + guard let tokenTypes = self.scanTypeDelimiter(scanner: scanner), tokenTypes.count > 0 else { return nil } @@ -97,44 +95,30 @@ public final class Lexer { } } - private func scanPayload(scanner: Scanner) -> String? { - var payload: String = "" - #if os(Linux) - var char: String? - #else - var char: NSString? - #endif + private func scanPayload(scanner: Scanner) -> String { let hexChars = "abcdef0123456789" - while scanner.scanCharacters(from: CharacterSet(charactersIn: hexChars), into: &char), - let char = char as String? { - payload.append(char) - } - return payload + let characterSet = Set(hexChars) + return scanner.scanCharacters(from: characterSet) ?? "" } private func scanTypeDelimiter(scanner: Scanner) -> [TokenType]? { - #if os(Linux) - var delimiters: String? - #else - var delimiters: NSString? - #endif - if scanner.scanCharacters(from: typeDelimiters, into: &delimiters), let delimiters = delimiters { - let delimiters = String(delimiters) - if delimiters.count > 1 { - // if we found a string, we discard other type delimiters because there are part of the string - let tokenString = TokenType.string - if let char = delimiters.first, tokenString.rawValue == String(char) { - scanner.scanLocation -= delimiters.count - 1 - return [tokenString] - } - } - // sometimes we found one or more nil list (-) next to the type delimiter - // in that case we'll return the delimiter and one or more `Token.null` - return delimiters.compactMap { character -> TokenType? in - TokenType(rawValue: String(character)) + guard let delimiters = scanner.scanCharacters(from: self.typeDelimiters) else { + return nil + } + + if delimiters.count > 1 { + // if we found a string, we discard other type delimiters because there are part of the string + let tokenString = TokenType.string + if let char = delimiters.first, tokenString.rawValue == String(char) { + scanner.moveOffset(by: -(delimiters.count - 1)) + return [tokenString] } } - return nil + // sometimes we found one or more nil list (-) next to the type delimiter + // in that case we'll return the delimiter and one or more `Token.null` + return delimiters.compactMap { character -> TokenType? in + TokenType(rawValue: String(character)) + } } private func scanToken(scanner: Scanner, @@ -252,19 +236,12 @@ public final class Lexer { scanner: Scanner, redacted: Bool, withoutBuildSpecificInformation: Bool) -> String? { - guard let value = Int(length) else { + guard let value = Int(length), let scannedResult = scanner.scan(count: value) else { print("error parsing string") return nil } - #if swift(>=5.0) - let start = String.Index(utf16Offset: scanner.scanLocation, in: scanner.string) - let end = String.Index(utf16Offset: scanner.scanLocation + value, in: scanner.string) - #else - let start = String.Index(encodedOffset: scanner.scanLocation) - let end = String.Index(encodedOffset: scanner.scanLocation + value) - #endif - scanner.scanLocation += value - var result = String(scanner.string[start.. 21 ? scanLocation + 21 : string.count - scanLocation - #if swift(>=5.0) - let start = String.Index(utf16Offset: scanLocation, in: self.string) - let end = String.Index(utf16Offset: endCount, in: self.string) - #else - let start = String.Index(encodedOffset: scanLocation) - let end = String.Index(encodedOffset: endCount) - #endif + let currentLocation = self.offset + let contentSize = self.string.count + + let start = String.Index(compilerSafeOffset: currentLocation, in: self.string) + let endCount = contentSize - currentLocation > 21 ? currentLocation + 21 : contentSize - currentLocation + let end = String.Index(compilerSafeOffset: endCount, in: self.string) + if end <= start { - return String(string[start..= self.stringEndIndex + } + + init(string: String) { + self.string = string + self.offset = 0 + } + + func scan(count: Int) -> String? { + let start = String.Index(compilerSafeOffset: self.offset, in: self.string) + let endOffset = self.offset + count + + guard endOffset <= self.string.utf16.count else { return nil } + + let end = String.Index(compilerSafeOffset: endOffset, in: self.string) + let result = self.string[start.. Bool { + guard self.string.starts(with: value) else { return false } + + self.offset += value.count + return true + } + + func scanCharacters(from allowedCharacters: Set) -> String? { + var prefix: String = "" + var characterIndex = String.Index(compilerSafeOffset: self.offset, in: self.string) + + while characterIndex < self.stringEndIndex { + let character = self.string[characterIndex] + + guard allowedCharacters.contains(character) else { + break + } + + prefix.append(character) + self.offset += 1 + characterIndex = String.Index(utf16Offset: self.offset, in: self.string) + } + + return prefix + } + + func moveOffset(by value: Int) { + self.offset += value + } +} diff --git a/Sources/XCLogParser/loglocation/LogLoader.swift b/Sources/XCLogParser/loglocation/LogLoader.swift index c7bfcd3..4465c57 100644 --- a/Sources/XCLogParser/loglocation/LogLoader.swift +++ b/Sources/XCLogParser/loglocation/LogLoader.swift @@ -36,7 +36,6 @@ public struct LogLoader { return String(cString: charPointer, encoding: .ascii) } - guard let contents = string else { throw LogError.readingFile(url.path) } diff --git a/Sources/XCLogParser/logmanifest/LogManifest.swift b/Sources/XCLogParser/logmanifest/LogManifest.swift index 57773c6..eafff2e 100644 --- a/Sources/XCLogParser/logmanifest/LogManifest.swift +++ b/Sources/XCLogParser/logmanifest/LogManifest.swift @@ -25,7 +25,7 @@ public struct LogManifest { public init() {} - public func getWithLogOptions(_ logOptions: LogOptions) throws -> [LogManifestEntry] { + public func getWithLogOptions(_ logOptions: LogOptions) throws -> [LogManifestEntry] { let logFinder = LogFinder() let logManifestURL = try logFinder.findLogManifestWithLogOptions(logOptions) let logManifestDictionary = try getDictionaryFromURL(logManifestURL) diff --git a/Sources/XCLogParser/parser/IDEActivityLogSection+Parsing.swift b/Sources/XCLogParser/parser/IDEActivityLogSection+Parsing.swift index b16a7b8..189a1eb 100644 --- a/Sources/XCLogParser/parser/IDEActivityLogSection+Parsing.swift +++ b/Sources/XCLogParser/parser/IDEActivityLogSection+Parsing.swift @@ -42,8 +42,7 @@ extension IDEActivityLogSection { func groupedByTarget() -> IDEActivityLogSection { // The only way to know if the structure is flatten is to check the first elements // for the `(in target 'ABC' from project Project)` string - let firstElements = subSections.prefix(15) // we only analyze up to the first 15 subsections - let isFlatten = firstElements.contains { $0.getTargetFromCommand() != nil } + let isFlatten = subSections.contains { $0.getTargetFromCommand() != nil } if isFlatten { let mainTarget = "$MainTarget" let targetsDictionary = subSections.reduce( diff --git a/Tests/XCLogParserTests/ActivityParserTests.swift b/Tests/XCLogParserTests/ActivityParserTests.swift index de1c787..dbb2aa8 100644 --- a/Tests/XCLogParserTests/ActivityParserTests.swift +++ b/Tests/XCLogParserTests/ActivityParserTests.swift @@ -97,7 +97,19 @@ class ActivityParserTests: XCTestCase { Token.string("501796C4-6BE4-4F80-9F9D-3269617ECC17"), Token.string("localizedResultString"), Token.string("xcbuildSignature"), - Token.list(1), + Token.list(3), + Token.classNameRef("IDEFoundation.IDEActivityLogSectionAttachment"), + Token.string("com.apple.dt.ActivityLogSectionAttachment.TaskBacktrace"), + Token.int(1), + Token.int(0), + // swiftlint:disable:next line_length + Token.json(#"[{"description":"'Planning Swift module ConcurrencyExtras (arm64)' had never run","category":{"ruleNeverBuilt":{}},"identifier":{"storage":{"task":{"_0":[0,80,50,58,116,97,114,103,101,116,45,67,111,110,99,117,114,114,101,110,99,121,69,120,116,114,97,115,45,101,102,52,50,51,48,52,53,57,52,98,102,56,53,50,102,52,51,56,101,102,55,99,51,97,49,51,54,98,50,99,57,48,100,102,56,55,49,56,97,102,50,98,57,100,51,97,97,99,48,100,48,100,99,97,50,50,98,52,99,50,57,99,50,45,58,66,101,116,97,32,68,101,98,117,103,58,51,99,57,97,99,57,53,50,98,52,99,56,49,100,57,99,99,49,55,100,49,97,102,52,55,49,97,48,52,53,101,56]}}},"frameKind":{"genericTask":{}}}]"#), + Token.classNameRef("IDEFoundation.IDEActivityLogSectionAttachment"), + Token.string("com.apple.dt.ActivityLogSectionAttachment.BuildOperationMetrics"), + Token.int(1), + Token.int(0), + // swiftlint:disable:next line_length + Token.json(#"{"clangCacheHits":0,"clangCacheMisses":2,"swiftCacheHits":0,"swiftCacheMisses":8}"#), Token.classNameRef("IDEFoundation.IDEActivityLogSectionAttachment"), Token.string("com.apple.dt.ActivityLogSectionAttachment.TaskMetrics"), Token.int(1), @@ -339,7 +351,11 @@ class ActivityParserTests: XCTestCase { XCTAssertEqual("501796C4-6BE4-4F80-9F9D-3269617ECC17", logSection.uniqueIdentifier) XCTAssertEqual("localizedResultString", logSection.localizedResultString) XCTAssertEqual("xcbuildSignature", logSection.xcbuildSignature) - XCTAssertEqual(1, logSection.attachments.count) + XCTAssertEqual(3, logSection.attachments.count) + XCTAssertEqual(logSection.attachments[0].backtrace?.frames.first?.category, .ruleNeverBuilt) + print(logSection.attachments) + XCTAssertEqual(logSection.attachments[1].buildOperationMetrics?.clangCacheMisses, 2) + XCTAssertEqual(logSection.attachments[2].metrics?.wcDuration, 1) XCTAssertEqual(0, logSection.unknown) } diff --git a/docs/Xcactivitylog Format.md b/docs/Xcactivitylog Format.md index dc6c31c..2869f64 100644 --- a/docs/Xcactivitylog Format.md +++ b/docs/Xcactivitylog Format.md @@ -155,4 +155,4 @@ Inside the logs you can find these classes: If you search for them, you will find that they belong to the IDEFoundation.framework. A private framework part of Xcode. You can class dump it to get the headers of those classes. Once you have the headers, you will have the name and type of the properties that belong to the class. Now, you can match them to the tokens you got from the log. Some of them are in the same order than in the headers, but for others it will be about trial and error. -In the project you can find those classes with their properties in the order in which they appear in the log in the file (IDEActivityModel.swift)[https://github.com/MobileNativeFoundation/XCLogParser/blob/master/Sources/XCLogParser/activityparser/IDEActivityModel.swift]. +In the project you can find those classes with their properties in the order in which they appear in the log in the file [IDEActivityModel.swift](../Sources/XCLogParser/activityparser/IDEActivityModel.swift).