massive refactoring for IC (#25282)

TODO:

- [ ] test writing of .nif files
- [x] implement loading of fields in PType/PSym that might not have been
loaded
- [ ] implement interface logic
- [ ] implement pragma "replays"
- [ ] implement special logic for `converter`
- [ ] implement special logic for `method`
- [ ] test the logic holds up for `export`
- [ ] implement logic to free the memory of PSym/PType if memory
pressure is high
- [ ] implement logic to close memory mapped files if too many are open.

---------

Co-authored-by: demotomohiro <gpuppur@gmail.com>
Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com>
Co-authored-by: Jacek Sieka <arnetheduck@gmail.com>

Author: 4 people

compiler/ast.nim

@@ -369,7 +369,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; n
     let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray
     if needsIndirect:
       n.typ() = n.typ.exactReplica
-      n.typ.flags.incl tfVarIsPtr
+      n.typ.incl tfVarIsPtr
     a = initLocExprSingleUse(p, n)
     a = withTmpIfNeeded(p, a, needsTmp)
     if needsIndirect: a.flags.incl lfIndirect

compiler/ast2nif.nim

@@ -3363,8 +3363,9 @@ proc genConstSetup(p: BProc; sym: PSym): bool =
   useHeader(m, sym)
   if sym.loc.k == locNone:
     fillBackendName(p.module, sym)
-    fillLoc(sym.loc, locData, sym.astdef, OnStatic)
-  if m.hcrOn: incl(sym.loc.flags, lfIndirect)
+    ensureMutable sym
+    fillLoc(sym.locImpl, locData, sym.astdef, OnStatic)
+  if m.hcrOn: incl(sym, lfIndirect)
   result = lfNoDecl notin sym.loc.flags
 
 proc genConstHeader(m, q: BModule; p: BProc, sym: PSym) =

compiler/astdef.nim

@@ -126,9 +126,10 @@ proc genVarTuple(p: BProc, n: PNode) =
     let vn = n[i]
     let v = vn.sym
     if sfCompileTime in v.flags: continue
+    ensureMutable v
     if sfGlobal in v.flags:
       assignGlobalVar(p, vn, "")
-      genObjectInit(p, cpsInit, v.typ, v.loc, constructObj)
+      genObjectInit(p, cpsInit, v.typ, v.locImpl, constructObj)
       registerTraverseProc(p, v)
     else:
       assignLocalVar(p, vn)
@@ -142,9 +143,9 @@ proc genVarTuple(p: BProc, n: PNode) =
         if t.n[i].kind != nkSym: internalError(p.config, n.info, "genVarTuple")
         mangleRecFieldName(p.module, t.n[i].sym)
     field.snippet = dotField(rtup, fieldName)
-    putLocIntoDest(p, v.loc, field)
+    putLocIntoDest(p, v.locImpl, field)
     if forHcr or isGlobalInBlock:
-      hcrGlobals.add((loc: v.loc, tp: CNil))
+      hcrGlobals.add((loc: v.locImpl, tp: CNil))
 
   if forHcr:
     # end the block where the tuple gets initialized
@@ -460,7 +461,8 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
   if value.kind != nkEmpty and valueAsRope.len == 0:
     genLineDir(targetProc, vn)
     if not isCppCtorCall:
-      loadInto(targetProc, vn, value, v.loc)
+      ensureMutable v
+      loadInto(targetProc, vn, value, v.locImpl)
   if forHcr:
     endBlockWith(targetProc):
       finishBranch(p.s(cpsStmts), hcrInit)
@@ -736,7 +738,8 @@ proc genBlock(p: BProc, n: PNode, d: var TLoc) =
       # named block?
       assert(n[0].kind == nkSym)
       var sym = n[0].sym
-      sym.loc.k = locOther
+      ensureMutable sym
+      sym.locImpl.k = locOther
       sym.position = p.breakIdx+1
     expr(p, n[1], d)
     endSimpleBlock(p, scope)
@@ -1250,7 +1253,8 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
           initElifBranch(p.s(cpsStmts), ifStmt, orExpr)
         if exvar != nil:
           fillLocalName(p, exvar.sym)
-          fillLoc(exvar.sym.loc, locTemp, exvar, OnStack)
+          ensureMutable exvar.sym
+          fillLoc(exvar.sym.locImpl, locTemp, exvar, OnStack)
           linefmt(p, cpsStmts, "$1 $2 = T$3_;$n", [getTypeDesc(p.module, exvar.sym.typ),
             rdLoc(exvar.sym.loc), rope(etmp+1)])
         # we handled the error:
@@ -1298,7 +1302,8 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
             if isImportedException(typeNode.typ, p.config):
               let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:`
               fillLocalName(p, exvar.sym)
-              fillLoc(exvar.sym.loc, locTemp, exvar, OnStack)
+              ensureMutable exvar.sym
+              fillLoc(exvar.sym.locImpl, locTemp, exvar, OnStack)
               startBlockWith(p):
                 lineCg(p, cpsStmts, "catch ($1& $2) {$n", [getTypeDesc(p.module, typeNode.typ), rdLoc(exvar.sym.loc)])
               genExceptBranchBody(t[i][^1])  # exception handler body will duplicated for every type
@@ -1389,7 +1394,8 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) =
         if t[i][j].isInfixAs():
           let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:`
           fillLocalName(p, exvar.sym)
-          fillLoc(exvar.sym.loc, locTemp, exvar, OnUnknown)
+          ensureMutable exvar.sym
+          fillLoc(exvar.sym.locImpl, locTemp, exvar, OnUnknown)
           startBlockWith(p):
             lineCg(p, cpsStmts, "catch ($1& $2) {$n", [getTypeDesc(p.module, t[i][j][1].typ), rdLoc(exvar.sym.loc)])
         else:

compiler/ccgcalls.nim

@@ -84,7 +84,8 @@ proc fillBackendName(m: BModule; s: PSym) =
     if m.hcrOn:
       result.add '_'
       result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config))
-    s.loc.snippet = result
+    ensureMutable s
+    s.locImpl.snippet = result
 
 proc fillParamName(m: BModule; s: PSym) =
   if s.loc.snippet == "":
@@ -107,7 +108,8 @@ proc fillParamName(m: BModule; s: PSym) =
     # and a function called in main or proxy uses `socket` as a parameter name.
     # That would lead to either needing to reload `proxy` or to overwrite the
     # executable file for the main module, which is running (or both!) -> error.
-    s.loc.snippet = res.rope
+    ensureMutable s
+    s.locImpl.snippet = res.rope
 
 proc fillLocalName(p: BProc; s: PSym) =
   assert s.kind in skLocalVars+{skTemp}
@@ -122,7 +124,8 @@ proc fillLocalName(p: BProc; s: PSym) =
     elif s.kind != skResult:
       result.add "_" & rope(counter+1)
     p.sigConflicts.inc(key)
-    s.loc.snippet = result
+    ensureMutable s
+    s.locImpl.snippet = result
 
 proc scopeMangledParam(p: BProc; param: PSym) =
   ## parameter generation only takes BModule, not a BProc, so we have to
@@ -155,9 +158,10 @@ proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
     else:
       break
   let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.elementType else: typ
+  ensureMutable typ
   if typ.loc.snippet == "":
-    typ.typeName(typ.loc.snippet)
-    typ.loc.snippet.add $sig
+    typ.typeName(typ.locImpl.snippet)
+    typ.locImpl.snippet.add $sig
   else:
     when defined(debugSigHashes):
       # check consistency:
@@ -300,12 +304,13 @@ proc addAbiCheck(m: BModule; t: PType, name: Rope) =
 
 
 proc fillResult(conf: ConfigRef; param: PNode, proctype: PType) =
-  fillLoc(param.sym.loc, locParam, param, "Result",
+  ensureMutable param.sym
+  fillLoc(param.sym.locImpl, locParam, param, "Result",
           OnStack)
   let t = param.sym.typ
   if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, proctype):
-    incl(param.sym.loc.flags, lfIndirect)
-    param.sym.loc.storage = OnUnknown
+    incl(param.sym.locImpl.flags, lfIndirect)
+    param.sym.locImpl.storage = OnUnknown
 
 proc typeNameOrLiteral(m: BModule; t: PType, literal: string): Rope =
   if t.sym != nil and sfImportc in t.sym.flags and t.sym.magic == mNone:
@@ -524,14 +529,15 @@ proc genMemberProcParams(m: BModule; prc: PSym, superCall, rettype, name, params
   var types, names, args: seq[string] = @[]
   if not isCtor:
     var this = t.n[1].sym
+    ensureMutable this
     fillParamName(m, this)
-    fillLoc(this.loc, locParam, t.n[1],
+    fillLoc(this.locImpl, locParam, t.n[1],
             this.paramStorageLoc)
     if this.typ.kind == tyPtr:
-      this.loc.snippet = "this"
+      this.locImpl.snippet = "this"
     else:
-      this.loc.snippet = "(*this)"
-    names.add this.loc.snippet
+      this.locImpl.snippet = "(*this)"
+    names.add this.locImpl.snippet
     types.add getTypeDescWeak(m, this.typ, check, dkParam)
 
   let firstParam = if isCtor: 1 else: 2
@@ -545,21 +551,22 @@ proc genMemberProcParams(m: BModule; prc: PSym, superCall, rettype, name, params
       else:
         descKind = dkRefParam
     var typ, name: string
+    ensureMutable param
     fillParamName(m, param)
-    fillLoc(param.loc, locParam, t.n[i],
+    fillLoc(param.locImpl, locParam, t.n[i],
             param.paramStorageLoc)
     if ccgIntroducedPtr(m.config, param, t.returnType) and descKind == dkParam:
       typ = getTypeDescWeak(m, param.typ, check, descKind) & "*"
-      incl(param.loc.flags, lfIndirect)
-      param.loc.storage = OnUnknown
+      incl(param.locImpl.flags, lfIndirect)
+      param.locImpl.storage = OnUnknown
     elif weakDep:
       typ = getTypeDescWeak(m, param.typ, check, descKind)
     else:
       typ = getTypeDescAux(m, param.typ, check, descKind)
     if sfNoalias in param.flags:
       typ.add("NIM_NOALIAS ")
 
-    name = param.loc.snippet
+    name = param.locImpl.snippet
     types.add typ
     names.add name
     if sfCodegenDecl notin param.flags:
@@ -601,14 +608,15 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
         else:
           descKind = dkRefParam
       if isCompileTimeOnly(param.typ): continue
+      ensureMutable param
       fillParamName(m, param)
-      fillLoc(param.loc, locParam, t.n[i],
+      fillLoc(param.locImpl, locParam, t.n[i],
               param.paramStorageLoc)
       var typ: Rope
       if ccgIntroducedPtr(m.config, param, t.returnType) and descKind == dkParam:
         typ = ptrType(getTypeDescWeak(m, param.typ, check, descKind))
-        incl(param.loc.flags, lfIndirect)
-        param.loc.storage = OnUnknown
+        incl(param.locImpl.flags, lfIndirect)
+        param.locImpl.storage = OnUnknown
       elif weakDep:
         typ = (getTypeDescWeak(m, param.typ, check, descKind))
       else:
@@ -620,9 +628,9 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
       var j = 0
       while arr.kind in {tyOpenArray, tyVarargs}:
         # this fixes the 'sort' bug:
-        if param.typ.kind in {tyVar, tyLent}: param.loc.storage = OnUnknown
+        if param.typ.kind in {tyVar, tyLent}: param.locImpl.storage = OnUnknown
         # need to pass hidden parameter:
-        params.addParam(paramBuilder, name = param.loc.snippet & "Len_" & $j, typ = NimInt)
+        params.addParam(paramBuilder, name = param.locImpl.snippet & "Len_" & $j, typ = NimInt)
         inc(j)
         arr = arr[0].skipTypes({tySink})
     if t.returnType != nil and isInvalidReturnType(m.config, t):
@@ -707,7 +715,8 @@ proc genRecordFieldsAux(m: BModule; n: PNode,
     if field.typ.kind == tyVoid: return
     #assert(field.ast == nil)
     let sname = mangleRecFieldName(m, field)
-    fillLoc(field.loc, locField, n, unionPrefix & sname, OnUnknown)
+    ensureMutable field
+    fillLoc(field.locImpl, locField, n, unionPrefix & sname, OnUnknown)
     # for importcpp'ed objects, we only need to set field.loc, but don't
     # have to recurse via 'getTypeDescAux'. And not doing so prevents problems
     # with heavily templatized C++ code:
@@ -1155,7 +1164,8 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool
   let isCtor = sfConstructor in prc.flags
   var check = initIntSet()
   fillBackendName(m, prc)
-  fillLoc(prc.loc, locProc, prc.ast[namePos], OnUnknown)
+  ensureMutable prc
+  fillLoc(prc.locImpl, locProc, prc.ast[namePos], OnUnknown)
   var memberOp = "#." #only virtual
   var typ: PType
   if isCtor:
@@ -1187,7 +1197,7 @@ proc genMemberProcHeader(m: BModule; prc: PSym; result: var Builder; asPtr: bool
     superCall = ""
   else:
     if not isCtor:
-      prc.loc.snippet = "$1$2(@)" % [memberOp, name]
+      prc.locImpl.snippet = "$1$2(@)" % [memberOp, name]
     elif superCall != "":
       superCall = " : " & superCall
 
@@ -1202,14 +1212,15 @@ proc genProcHeader(m: BModule; prc: PSym; result: var Builder; visibility: var D
   # using static is needed for inline procs
   var check = initIntSet()
   fillBackendName(m, prc)
-  fillLoc(prc.loc, locProc, prc.ast[namePos], OnUnknown)
+  ensureMutable prc
+  fillLoc(prc.locImpl, locProc, prc.ast[namePos], OnUnknown)
   var rettype: Snippet = ""
   var desc = newBuilder("")
   genProcParams(m, prc.typ, rettype, desc, check, true, false)
   let params = extract(desc)
   # handle the 2 options for hotcodereloading codegen - function pointer
   # (instead of forward declaration) or header for function body with "_actual" postfix
-  var name = prc.loc.snippet
+  var name = prc.locImpl.snippet
   if not asPtr and isReloadable(m, prc):
     name.add("_actual")
   # careful here! don't access ``prc.ast`` as that could reload large parts of
@@ -1449,7 +1460,7 @@ proc genObjectInfo(m: BModule; typ, origType: PType, name: Rope; info: TLineInfo
   var t = typ.baseClass
   while t != nil:
     t = t.skipTypes(skipPtrs)
-    t.flags.incl tfObjHasKids
+    t.incl tfObjHasKids
     t = t.baseClass
 
 proc genTupleInfo(m: BModule; typ, origType: PType, name: Rope; info: TLineInfo) =
@@ -1645,8 +1656,8 @@ proc generateRttiDestructor(g: ModuleGraph; typ: PType; owner: PSym; kind: TType
   n[bodyPos] = body
   result.ast = n
 
-  incl result.flags, sfFromGeneric
-  incl result.flags, sfGeneratedOp
+  incl result.flagsImpl, sfFromGeneric
+  incl result.flagsImpl, sfGeneratedOp
 
 proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp; result: var Builder) =
   let theProc = getAttachedOp(m.g.graph, t, op)