Prepare to switch on new lookup code (#2701)

* Beginnings of grandparent override scoping

* intermediate

* implement basic overrides, constructors still broken

* intermediate

* this seems to work much better

* dartfmt

* This now seems to work with tests

* rebuild && dartfmt

* fix up tests a bit

* add comment

* fix package / library cases

* move test to correct group

* remove assert

* switch it on

* bare prefix changes

* starting to work on test problems

* Fix some errors in the tests

* correct tests

* switch it off... one more PR ;-)

* Patch up extension method test, almost forgot why we bothered with this

* class counter increment

Author: jcollins-g

lib/src/dartdoc_options.dart

@@ -1289,8 +1289,8 @@ class DartdocOptionContext extends DartdocOptionContextBase
   List<String> get excludePackages =>
       optionSet['excludePackages'].valueAt(context);
 
-  bool get experimentalReferenceLookup =>
-      optionSet['experimentalReferenceLookup'].valueAt(context);
+  bool get enhancedReferenceLookup =>
+      optionSet['enhancedReferenceLookup'].valueAt(context);
 
   String get flutterRoot => optionSet['flutterRoot'].valueAt(context);
 
@@ -1426,10 +1426,13 @@ Future<List<DartdocOption<Object>>> createDartdocOptions(
     DartdocOptionArgOnly<List<String>>('excludePackages', [], resourceProvider,
         help: 'Package names to ignore.', splitCommas: true),
     DartdocOptionArgFile<bool>(
-        'experimentalReferenceLookup', false, resourceProvider,
+        'enhancedReferenceLookup', false, resourceProvider,
         hide: true,
         help:
-            'Use an experimental code path to resolve comment reference lookups',
+            'Use the new comment reference lookup system instead of the legacy '
+            'version.  Please file a bug referencing this flag if you have to '
+            'change it to false -- this flag and associated behavior will '
+            'disappear in a future version.',
         negatable: true),
     // This could be a ArgOnly, but trying to not provide too many ways
     // to set the flutter root.

lib/src/generator/templates.runtime_renderers.dart

@@ -14694,7 +14694,7 @@ const _invisibleGetters = {
     'examplePathPrefix',
     'exclude',
     'excludePackages',
-    'experimentalReferenceLookup',
+    'enhancedReferenceLookup',
     'flutterRoot',
     'hideSdkText',
     'include',

lib/src/markdown_processor.dart

@@ -380,7 +380,10 @@ MatchingLinkResult _getMatchingLinkElementLegacy(
   // Try expensive not-scoped lookup.
   if (refModelElement == null && warnable is ModelElement) {
     Container preferredClass = _getPreferredClass(warnable);
-    if (preferredClass is Extension) {
+    // We might still get here in comparison mode, don't complain if that's
+    // the case.
+    if (preferredClass is Extension &&
+        warnable.config.enhancedReferenceLookup == false) {
       warnable.warn(PackageWarning.notImplemented,
           message:
               'Comment reference resolution inside extension methods is not yet implemented');
@@ -1032,8 +1035,8 @@ md.Node _makeLinkNode(String codeRef, Warnable warnable) {
 
 @visibleForTesting
 MatchingLinkResult getMatchingLinkElement(Warnable warnable, String codeRef,
-    {bool experimentalReferenceLookup}) {
-  experimentalReferenceLookup ??= warnable.config.experimentalReferenceLookup;
+    {bool enhancedReferenceLookup}) {
+  enhancedReferenceLookup ??= warnable.config.enhancedReferenceLookup;
   MatchingLinkResult result, resultOld, resultNew;
   // Do a comparison between result types only if the warnings for them are
   // enabled, because there's a significant performance penalty.
@@ -1046,12 +1049,12 @@ MatchingLinkResult getMatchingLinkElement(Warnable warnable, String codeRef,
     if (resultNew.commentReferable != null) {
       markdownStats.resolvedNewLookupReferences++;
     }
-    result = experimentalReferenceLookup ? resultNew : resultOld;
+    result = enhancedReferenceLookup ? resultNew : resultOld;
     if (resultOld.commentReferable != null) {
       markdownStats.resolvedOldLookupReferences++;
     }
   } else {
-    if (experimentalReferenceLookup) {
+    if (enhancedReferenceLookup) {
       result = _getMatchingLinkElementCommentReferable(codeRef, warnable);
     } else {
       result = _getMatchingLinkElementLegacy(codeRef, warnable);

lib/src/model/prefix.dart

@@ -4,6 +4,7 @@
 
 import 'package:analyzer/dart/element/element.dart';
 import 'package:analyzer/dart/element/scope.dart';
+import 'package:analyzer/src/dart/element/element.dart';
 import 'package:dartdoc/src/model/comment_referable.dart';
 import 'package:dartdoc/src/model/library.dart';
 
@@ -12,21 +13,34 @@ import '../../dartdoc.dart';
 /// Represents a [PrefixElement] for dartdoc.
 ///
 /// Like [Parameter], it doesn't have doc pages, but participates in lookups.
+/// Forwards to its referenced library if referred to directly.
 class Prefix extends ModelElement implements EnclosedElement {
   /// [library] is the library the prefix is defined in, not the [Library]
   /// referred to by the [PrefixElement].
   Prefix(PrefixElement element, Library library, PackageGraph packageGraph)
       : super(element, library, packageGraph);
 
   @override
-  // TODO(jcollins-g): consider allowing bare prefixes to link to a library doc?
   bool get isCanonical => false;
 
+  Library _associatedLibrary;
+  // TODO(jcollins-g): consider connecting PrefixElement to the imported library
+  // in analyzer?
+  Library get associatedLibrary =>
+      _associatedLibrary ??= ModelElement.fromElement(
+          library.element.imports
+              .firstWhere((i) => i.prefix == element)
+              .importedLibrary,
+          packageGraph);
+
+  @override
+  Library get canonicalModelElement => associatedLibrary.canonicalLibrary;
+
   @override
   Scope get scope => element.scope;
 
   @override
-  PrefixElement get element => super.element;
+  PrefixElementImpl get element => super.element;
 
   @override
   ModelElement get enclosingElement => library;
@@ -36,7 +50,8 @@ class Prefix extends ModelElement implements EnclosedElement {
       throw UnimplementedError('prefixes have no generated files in dartdoc');
 
   @override
-  String get href => null;
+  String get href =>
+      canonicalModelElement == null ? null : canonicalModelElement.href;
 
   @override
   String get kind => 'prefix';

test/end2end/model_test.dart

@@ -1897,7 +1897,7 @@ void main() {
     });
 
     test('correctly finds all the classes', () {
-      expect(classes, hasLength(38));
+      expect(classes, hasLength(39));
     });
 
     test('abstract', () {
@@ -2140,10 +2140,10 @@ void main() {
 
     MatchingLinkResult originalLookup(Warnable element, String codeRef) =>
         definingLinkResult(getMatchingLinkElement(element, codeRef,
-            experimentalReferenceLookup: false));
+            enhancedReferenceLookup: false));
     MatchingLinkResult newLookup(Warnable element, String codeRef) =>
         definingLinkResult(getMatchingLinkElement(element, codeRef,
-            experimentalReferenceLookup: true));
+            enhancedReferenceLookup: true));
 
     MatchingLinkResult bothLookup(Warnable element, String codeRef) {
       var originalLookupResult = originalLookup(element, codeRef);
@@ -2254,8 +2254,11 @@ void main() {
 
     group('Ordinary namespace cases', () {
       Package DartPackage;
-      Library Dart;
-      ModelFunction doesStuff, function1, topLevelFunction;
+      Library Dart, mylibpub;
+      ModelFunction doesStuff,
+          function1,
+          topLevelFunction,
+          aFunctionUsingRenamedLib;
       TopLevelVariable incorrectDocReference,
           incorrectDocReferenceFromEx,
           nameWithTwoUnderscores,
@@ -2281,6 +2284,10 @@ void main() {
           aConstructorShadowedField;
 
       setUpAll(() async {
+        mylibpub = packageGraph.allLibraries.values
+            .firstWhere((l) => l.name == 'mylibpub');
+        aFunctionUsingRenamedLib = fakeLibrary.functions
+            .firstWhere((f) => f.name == 'aFunctionUsingRenamedLib');
         Dart = packageGraph.allLibraries.values
             .firstWhere((l) => l.name == 'Dart');
         DartPackage = packageGraph.packages.firstWhere((p) => p.name == 'Dart');
@@ -2356,6 +2363,16 @@ void main() {
             .firstWhere((f) => f.name == 'aConstructorShadowed');
       });
 
+      test('Referring to a renamed library directly works', () {
+        // The new code forwards from the prefix, so doesn't quite work the
+        // same as the old for an equality comparison via [MatchingLinkResult].
+        expect(
+            (bothLookup(aFunctionUsingRenamedLib, 'renamedLib').commentReferable
+                    as ModelElement)
+                .canonicalModelElement,
+            equals(mylibpub));
+      });
+
       test('Referring to libraries and packages with the same name is fine',
           () {
         expect(bothLookup(Apple, 'Dart'), equals(MatchingLinkResult(Dart)));
@@ -2660,20 +2677,20 @@ void main() {
           megaTron.potentiallyApplicableExtensions, orderedEquals([arm, leg]));
     });
 
-    // TODO(jcollins-g): implement feature and update tests
     test('documentation links do not crash in base cases', () {
-      packageGraph.packageWarningCounter.hasWarning(
-          doStuff,
-          PackageWarning.notImplemented,
-          'Comment reference resolution inside extension methods is not yet implemented');
-      packageGraph.packageWarningCounter.hasWarning(
-          doSomeStuff,
-          PackageWarning.notImplemented,
-          'Comment reference resolution inside extension methods is not yet implemented');
+      // Parameters are OK.
       expect(doStuff.documentationAsHtml, contains('<code>another</code>'));
-      expect(doSomeStuff.documentationAsHtml,
-          contains('<code>String.extensionNumber</code>'));
-    });
+      expect(
+          doStuff.documentationAsHtml,
+          contains(
+              '<a href="%%__HTMLBASE_dartdoc_internal__%%ex/AnExtendableThing/aMember.html">aMember</a>'));
+      expect(
+          doStuff.documentationAsHtml,
+          contains(
+              '<a href="%%__HTMLBASE_dartdoc_internal__%%ex/AnExtendableThing-class.html">AnExtendableThing</a>'));
+      // TODO(jcollins-g): consider linking via applied extensions?
+      expect(doSomeStuff.documentationAsHtml, contains('<code>aMember</code>'));
+    }, skip: 'unskip when enhanced lookups are on by default');
 
     test(
         'references from outside an extension refer correctly to the extension',

testing/test_package/lib/example.dart

@@ -610,22 +610,22 @@ extension AnExtension<Q> on WithGeneric<Q> {
   int call(String s) => 0;
 }
 
-extension SimpleStringExtension on String {
+class AnExtendableThing {
+  int aMember;
+}
+
+extension SimpleStringExtension on AnExtendableThing {
   /// Print this and [another].
-  /// Refer to [indexOf], from [String].
+  /// Refer to [aMember], from [AnExtendableThing].
   /// Also refer to [extensionNumber].
-  void doStuff(String another) {
-    print(this + another);
-  }
+  void doStuff(String another) {}
 
   int get extensionNumber => 3;
 }
 
 class ExtensionUser {
-  /// Refer to [String.extensionNumber], which we use here.
-  void doSomeStuff(String things) {
-    print(things.extensionNumber + 1);
-  }
+  /// Refer to [aMember], which we use here.
+  void doSomeStuff(String things) {}
 }
 
 /// Extension on List