integrate visualfsharp master

Author: dsyme

fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj

@@ -278,10 +278,6 @@
     <Compile Include="$(FSharpSourcesRoot)/ilx/EraseUnions.fs">
       <Link>ILXErase/EraseUnions.fs</Link>
     </Compile>
-    <FsLex Include="$(FSharpSourcesRoot)\absil\illex.fsl">
-      <OtherFlags>--unicode --lexlib Internal.Utilities.Text.Lexing</OtherFlags>
-      <Link>AbsIL/illex.fsl</Link>
-    </FsLex>
     <FsLex Include="$(FSharpSourcesRoot)\fsharp\lex.fsl">
       <OtherFlags>--unicode --lexlib Internal.Utilities.Text.Lexing</OtherFlags>
       <Link>ParserAndUntypedAST/lex.fsl</Link>

src/absil/ilread.fs

@@ -33,9 +33,11 @@ open System.Reflection
 
 let checking = false  
 let logging = false
-let _ = if checking then dprintn "warning : Ilread.checking is on"
+let _ = if checking then dprintn "warning : ILBinaryReader.checking is on"
 let noStableFileHeuristic = try (System.Environment.GetEnvironmentVariable("FSharp_NoStableFileHeuristic") <> null) with _ -> false
 let alwaysMemoryMapFSC = try (System.Environment.GetEnvironmentVariable("FSharp_AlwaysMemoryMapCommandLineCompiler") <> null) with _ -> false
+let stronglyHeldReaderCacheSizeDefault = 30
+let stronglyHeldReaderCacheSize = try (match System.Environment.GetEnvironmentVariable("FSharp_StronglyHeldBinaryReaderCacheSize") with null -> stronglyHeldReaderCacheSizeDefault | s -> int32 s) with _ -> stronglyHeldReaderCacheSizeDefault
 
 let singleOfBits (x:int32) = System.BitConverter.ToSingle(System.BitConverter.GetBytes(x), 0)
 let doubleOfBits (x:int64) = System.BitConverter.Int64BitsToDouble(x)
@@ -346,7 +348,7 @@ type ByteFile(fileName: string, bytes:byte[]) =
 /// This is the default implementation used by F# Compiler Services when accessing "stable" binaries.  It is not used
 /// by Visual Studio, where tryGetMetadataSnapshot provides a RawMemoryFile backed by Roslyn data.
 [<DebuggerDisplay("{FileName}")>]
-type WeakByteFile(fileName: string) = 
+type WeakByteFile(fileName: string, chunk: (int * int) option) = 
 
     do stats.weakByteFileCount <- stats.weakByteFileCount + 1
 
@@ -357,30 +359,29 @@ type WeakByteFile(fileName: string) =
     let weakBytes = new WeakReference<byte[]> (null)
 
     member __.FileName = fileName
-    /// Get the bytes for the file
-    member this.Get() =
-        let mutable tg = null
-        if not (weakBytes.TryGetTarget(&tg)) then 
-            if FileSystem.GetLastWriteTimeShim(fileName) <> fileStamp then 
-               errorR (Error (FSComp.SR.ilreadFileChanged fileName, range0))
-
-            tg <- FileSystem.ReadAllBytesShim fileName
-            weakBytes.SetTarget tg
-        tg
 
+    /// Get the bytes for the file
     interface BinaryFile with
-        override __.GetView() = 
-            let mutable tg = null
+
+        override this.GetView() = 
             let strongBytes = 
+                let mutable tg = null
                 if not (weakBytes.TryGetTarget(&tg)) then 
                     if FileSystem.GetLastWriteTimeShim(fileName) <> fileStamp then 
-                       errorR (Error (FSComp.SR.ilreadFileChanged fileName, range0))
+                        error (Error (FSComp.SR.ilreadFileChanged fileName, range0))
+
+                    let bytes = 
+                        match chunk with 
+                        | None -> FileSystem.ReadAllBytesShim fileName
+                        | Some(start, length) -> File.ReadBinaryChunk (fileName, start, length)
+
+                    tg <- bytes
+
+                    weakBytes.SetTarget bytes
 
-                    tg <- FileSystem.ReadAllBytesShim fileName
-                    weakBytes.SetTarget tg
                 tg
-            (ByteView(strongBytes) :> BinaryView)
 
+            (ByteView(strongBytes) :> BinaryView)
 
 
 let seekReadByte (mdv:BinaryView) addr = mdv.ReadByte addr
@@ -3927,27 +3928,31 @@ type ILModuleReader(ilModule: ILModuleDef, ilAssemblyRefs: Lazy<ILAssemblyRef li
 
 // ++GLOBAL MUTABLE STATE (concurrency safe via locking)
 type ILModuleReaderCacheLockToken() = interface LockToken
-let ilModuleReaderCache = new AgedLookup<ILModuleReaderCacheLockToken, (string * System.DateTime * ILScopeRef * bool * ReduceMemoryFlag * MetadataOnlyFlag), ILModuleReader>(0, areSimilar=(fun (x, y) -> x = y))
+type ILModuleReaderCacheKey = ILModuleReaderCacheKey of string * DateTime * ILScopeRef * bool * ReduceMemoryFlag * MetadataOnlyFlag
+let ilModuleReaderCache = new AgedLookup<ILModuleReaderCacheLockToken, ILModuleReaderCacheKey, ILModuleReader>(stronglyHeldReaderCacheSize, areSimilar=(fun (x, y) -> x = y))
 let ilModuleReaderCacheLock = Lock()
 
 let stableFileHeuristicApplies fileName = 
     not noStableFileHeuristic && try FileSystem.IsStableFileHeuristic fileName with _ -> false
 
-let createByteFile opts fileName = 
+let createByteFileChunk opts fileName chunk = 
     // If we're trying to reduce memory usage then we are willing to go back and re-read the binary, so we can use
     // a weakly-held handle to an array of bytes.
     if opts.reduceMemoryUsage = ReduceMemoryFlag.Yes && stableFileHeuristicApplies fileName then 
-        WeakByteFile(fileName) :> BinaryFile 
+        WeakByteFile(fileName, chunk) :> BinaryFile 
     else 
-        let bytes = FileSystem.ReadAllBytesShim(fileName)
+        let bytes = 
+            match chunk with 
+            | None -> FileSystem.ReadAllBytesShim fileName
+            | Some (start, length) -> File.ReadBinaryChunk(fileName, start, length)
         ByteFile(fileName, bytes) :> BinaryFile
 
-let tryMemoryMap opts fileName = 
+let tryMemoryMapWholeFile opts fileName = 
     let file = 
         try 
             MemoryMapFile.Create fileName :> BinaryFile
         with _ ->
-            createByteFile opts fileName
+            createByteFileChunk opts fileName None
     let disposer = 
         { new IDisposable with 
            member __.Dispose() = 
@@ -3963,17 +3968,16 @@ let OpenILModuleReaderFromBytes fileName bytes opts =
 
 let OpenILModuleReader fileName opts = 
     // Pseudo-normalize the paths.
-    let ((_,writeStamp,_,_,_,_) as key), keyOk = 
+    let (ILModuleReaderCacheKey (fullPath,writeStamp,_,_,_,_) as key), keyOk = 
         try 
-           (FileSystem.GetFullPathShim(fileName), 
-            FileSystem.GetLastWriteTimeShim(fileName), 
-            opts.ilGlobals.primaryAssemblyScopeRef, 
-            opts.pdbPath.IsSome,
-            opts.reduceMemoryUsage,
-            opts.metadataOnly), true
-        with e -> 
-            System.Diagnostics.Debug.Assert(false, sprintf "Failed to compute key in OpenILModuleReader cache for '%s'. Falling back to uncached." fileName) 
-            ("", System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes), false
+           let fullPath = FileSystem.GetFullPathShim(fileName)
+           let writeTime = FileSystem.GetLastWriteTimeShim(fileName)
+           let key = ILModuleReaderCacheKey (fullPath, writeTime, opts.ilGlobals.primaryAssemblyScopeRef, opts.pdbPath.IsSome, opts.reduceMemoryUsage, opts.metadataOnly)
+           key, true
+        with exn -> 
+            System.Diagnostics.Debug.Assert(false, sprintf "Failed to compute key in OpenILModuleReader cache for '%s'. Falling back to uncached. Error = %s" fileName (exn.ToString())) 
+            let fakeKey = ILModuleReaderCacheKey(fileName, System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes)
+            fakeKey, false
 
     let cacheResult = 
         if keyOk then 
@@ -3999,30 +4003,29 @@ let OpenILModuleReader fileName opts =
 
                 // See if tryGetMetadata gives us a BinaryFile for the metadata section alone.
                 let mdfileOpt = 
-                    match opts.tryGetMetadataSnapshot (fileName, writeStamp) with 
-                    | Some (obj, start, len) -> Some (RawMemoryFile(fileName, obj, start, len) :> BinaryFile)
+                    match opts.tryGetMetadataSnapshot (fullPath, writeStamp) with 
+                    | Some (obj, start, len) -> Some (RawMemoryFile(fullPath, obj, start, len) :> BinaryFile)
                     | None  -> None
 
                 // For metadata-only, always use a temporary, short-lived PE file reader, preferably over a memory mapped file.
                 // Then use the metadata blob as the long-lived memory resource.
-                let disposer, pefileEager = tryMemoryMap opts fileName
+                let disposer, pefileEager = tryMemoryMapWholeFile opts fullPath
                 use _disposer = disposer
-                let (metadataPhysLoc, metadataSize, peinfo, pectxtEager, pevEager, _pdb) = openPEFileReader (fileName, pefileEager, None) 
+                let (metadataPhysLoc, metadataSize, peinfo, pectxtEager, pevEager, _pdb) = openPEFileReader (fullPath, pefileEager, None) 
                 let mdfile = 
                     match mdfileOpt with 
                     | Some mdfile -> mdfile
                     | None -> 
                         // If tryGetMetadata doesn't give anything, then just read the metadata chunk out of the binary
-                        let bytes = File.ReadBinaryChunk (fileName, metadataPhysLoc, metadataSize)
-                        ByteFile(fileName, bytes) :> BinaryFile
+                        createByteFileChunk opts fullPath (Some (metadataPhysLoc, metadataSize))
 
-                let ilModule, ilAssemblyRefs = openPEMetadataOnly (fileName, peinfo, pectxtEager, pevEager, mdfile, reduceMemoryUsage, opts.ilGlobals) 
+                let ilModule, ilAssemblyRefs = openPEMetadataOnly (fullPath, peinfo, pectxtEager, pevEager, mdfile, reduceMemoryUsage, opts.ilGlobals) 
                 new ILModuleReader(ilModule, ilAssemblyRefs, ignore)
             else
                 // If we are not doing metadata-only, then just go ahead and read all the bytes and hold them either strongly or weakly
                 // depending on the heuristic
-                let pefile = createByteFile opts fileName
-                let ilModule, ilAssemblyRefs, _pdb = openPE (fileName, pefile, None, reduceMemoryUsage, opts.ilGlobals) 
+                let pefile = createByteFileChunk opts fullPath None
+                let ilModule, ilAssemblyRefs, _pdb = openPE (fullPath, pefile, None, reduceMemoryUsage, opts.ilGlobals) 
                 new ILModuleReader(ilModule, ilAssemblyRefs, ignore)
 
         if keyOk then 
@@ -4042,14 +4045,14 @@ let OpenILModuleReader fileName opts =
         // multi-proc build. So use memory mapping, but only for stable files.  Other files
         // fill use an in-memory ByteFile
         let _disposer, pefile = 
-            if alwaysMemoryMapFSC || stableFileHeuristicApplies fileName then 
-                tryMemoryMap opts fileName
+            if alwaysMemoryMapFSC || stableFileHeuristicApplies fullPath then 
+                tryMemoryMapWholeFile opts fullPath
             else
-                let pefile = createByteFile opts fileName
+                let pefile = createByteFileChunk opts fullPath None
                 let disposer = { new IDisposable with member __.Dispose() = () }
                 disposer, pefile
 
-        let ilModule, ilAssemblyRefs, pdb = openPE (fileName, pefile, opts.pdbPath, reduceMemoryUsage, opts.ilGlobals)
+        let ilModule, ilAssemblyRefs, pdb = openPE (fullPath, pefile, opts.pdbPath, reduceMemoryUsage, opts.ilGlobals)
         let ilModuleReader = new ILModuleReader(ilModule, ilAssemblyRefs, (fun () -> ClosePdbReader pdb))
 
         // Readers with PDB reader disposal logic don't go in the cache.  Note the PDB reader is only used in static linking.