|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Checks that a published version of the four packages points at pages that exist. |
| 3 | +
|
| 4 | +For each of AustinHarris.JsonRpc, .AspNetCore, .SystemTextJson and .Newtonsoft at the given version, the script |
| 5 | +downloads the .nupkg from nuget.org, reads projectUrl, the repository url, releaseNotes and the package README out |
| 6 | +of the .nuspec, and requests every absolute http(s) URL found in them, expecting a 2xx. It then builds a scratch |
| 7 | +console project against the core package, prints the ObsoleteAttribute on Handler.RegisterFuction, |
| 8 | +Handler.UnRegisterFunction and Config.SetBeforeProcessHandler as the published assembly carries it, formats each |
| 9 | +UrlFormat with its DiagnosticId, requests the page and asserts the anchor id is in the HTML. |
| 10 | +
|
| 11 | +Usage: python3 .github/scripts/check_package_links.py 2.0.0-preview.3 [--keep] (exit 1 on any failure) |
| 12 | +
|
| 13 | +The check runs against nuget.org, so it is a release step (after the publish workflow has run and the flat |
| 14 | +container lists the version, which lags the push by a few minutes), not a pull-request gate. |
| 15 | +""" |
| 16 | +import io |
| 17 | +import os |
| 18 | +import re |
| 19 | +import shutil |
| 20 | +import subprocess |
| 21 | +import sys |
| 22 | +import tempfile |
| 23 | +import urllib.error |
| 24 | +import urllib.request |
| 25 | +import xml.etree.ElementTree as ET |
| 26 | +import zipfile |
| 27 | + |
| 28 | +PACKAGES = ["AustinHarris.JsonRpc", "AustinHarris.JsonRpc.AspNetCore", "AustinHarris.JsonRpc.SystemTextJson", |
| 29 | + "AustinHarris.JsonRpc.Newtonsoft"] |
| 30 | +FLAT = "https://api.nuget.org/v3-flatcontainer/{id}/{version}/{id}.{version}.nupkg" |
| 31 | +OBSOLETE = { # member -> expected diagnostic id |
| 32 | + "Handler.RegisterFuction": "JSONRPC0002", |
| 33 | + "Handler.UnRegisterFunction": "JSONRPC0003", |
| 34 | + "Config.SetBeforeProcessHandler": "JSONRPC0001", |
| 35 | +} |
| 36 | +URL = re.compile(r"https?://[^\s<>()\"'`\]]+") |
| 37 | +# Sample addresses in the READMEs (a host the reader runs) are not links to check. |
| 38 | +LOCAL = re.compile(r"^https?://(localhost|127\.0\.0\.1|\[::1\]|[^/]+\.(test|example|localhost)|example\.(com|org|net))(:\d+)?(/|$)", re.I) |
| 39 | +HEADERS = {"User-Agent": "JSON-RPC.NET package-link check (+https://github.com/Astn/JSON-RPC.NET)"} |
| 40 | + |
| 41 | +PROGRAM = r""" |
| 42 | +using System; |
| 43 | +using System.Linq; |
| 44 | +using System.Reflection; |
| 45 | +using AustinHarris.JsonRpc; |
| 46 | +
|
| 47 | +static class Program |
| 48 | +{ |
| 49 | + static void Main() |
| 50 | + { |
| 51 | + const BindingFlags All = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance; |
| 52 | + foreach (var m in typeof(Handler).GetMethods(All).Where(m => m.Name == "RegisterFuction" || m.Name == "UnRegisterFunction") |
| 53 | + .Concat(typeof(Config).GetMethods(All).Where(m => m.Name == "SetBeforeProcessHandler"))) |
| 54 | + foreach (var a in m.GetCustomAttributes<ObsoleteAttribute>()) |
| 55 | + Console.WriteLine(m.DeclaringType.Name + "." + m.Name + "|" + a.DiagnosticId + "|" + a.UrlFormat + "|" + a.Message); |
| 56 | + Console.WriteLine("assembly|" + typeof(Handler).Assembly.GetName().FullName); |
| 57 | + } |
| 58 | +} |
| 59 | +""" |
| 60 | + |
| 61 | +PROJECT = """<Project Sdk="Microsoft.NET.Sdk"> |
| 62 | + <PropertyGroup> |
| 63 | + <OutputType>Exe</OutputType> |
| 64 | + <TargetFramework>net8.0</TargetFramework> |
| 65 | + <Nullable>disable</Nullable> |
| 66 | + </PropertyGroup> |
| 67 | + <ItemGroup> |
| 68 | + <PackageReference Include="AustinHarris.JsonRpc" Version="{version}" /> |
| 69 | + </ItemGroup> |
| 70 | +</Project> |
| 71 | +""" |
| 72 | + |
| 73 | +NUGET_CONFIG = """<?xml version="1.0" encoding="utf-8"?> |
| 74 | +<configuration> |
| 75 | + <packageSources> |
| 76 | + <clear /> |
| 77 | + <add key="nuget.org" value="https://api.nuget.org/v3/index.json" /> |
| 78 | + </packageSources> |
| 79 | +</configuration> |
| 80 | +""" |
| 81 | + |
| 82 | +failures = [] |
| 83 | + |
| 84 | + |
| 85 | +def fail(message): |
| 86 | + failures.append(message) |
| 87 | + print("FAIL " + message) |
| 88 | + |
| 89 | + |
| 90 | +def fetch(url, binary=False): |
| 91 | + req = urllib.request.Request(url, headers=HEADERS) |
| 92 | + with urllib.request.urlopen(req, timeout=60) as r: |
| 93 | + data = r.read() |
| 94 | + return (r.status, data if binary else data.decode("utf-8", "replace")) |
| 95 | + |
| 96 | + |
| 97 | +def status(url): |
| 98 | + try: |
| 99 | + return fetch(url)[0] |
| 100 | + except urllib.error.HTTPError as e: |
| 101 | + return e.code |
| 102 | + except Exception as e: # DNS, TLS, timeout |
| 103 | + return str(e) |
| 104 | + |
| 105 | + |
| 106 | +def text(node): |
| 107 | + return (node.text or "").strip() if node is not None else "" |
| 108 | + |
| 109 | + |
| 110 | +def strip_trailing(url): |
| 111 | + return url.rstrip(".,;:!?)") |
| 112 | + |
| 113 | + |
| 114 | +def check_package(package, version, workdir): |
| 115 | + print("== " + package + " " + version) |
| 116 | + url = FLAT.format(id=package.lower(), version=version.lower()) |
| 117 | + try: |
| 118 | + code, data = fetch(url, binary=True) |
| 119 | + except urllib.error.HTTPError as e: |
| 120 | + fail("%s: nupkg download %s -> %s" % (package, url, e.code)) |
| 121 | + return |
| 122 | + path = os.path.join(workdir, package + "." + version + ".nupkg") |
| 123 | + with open(path, "wb") as f: |
| 124 | + f.write(data) |
| 125 | + z = zipfile.ZipFile(io.BytesIO(data)) |
| 126 | + spec_name = [n for n in z.namelist() if n.endswith(".nuspec")][0] |
| 127 | + root = ET.fromstring(z.read(spec_name)) |
| 128 | + ns = {"n": root.tag[1:].split("}")[0]} if root.tag.startswith("{") else {} |
| 129 | + meta = root.find("n:metadata", ns) if ns else root.find("metadata") |
| 130 | + |
| 131 | + def field(name): |
| 132 | + return text(meta.find("n:" + name, ns) if ns else meta.find(name)) |
| 133 | + |
| 134 | + spec_version = field("version") |
| 135 | + if spec_version != version: |
| 136 | + fail("%s: nuspec version %s, expected %s" % (package, spec_version, version)) |
| 137 | + project_url = field("projectUrl") |
| 138 | + repo = meta.find("n:repository", ns) if ns else meta.find("repository") |
| 139 | + repo_url = repo.get("url") if repo is not None else "" |
| 140 | + notes = field("releaseNotes") |
| 141 | + readme_name = field("readme") |
| 142 | + if not project_url: |
| 143 | + fail(package + ": no projectUrl") |
| 144 | + if not repo_url: |
| 145 | + fail(package + ": no repository url") |
| 146 | + if not notes: |
| 147 | + fail(package + ": no releaseNotes") |
| 148 | + if not readme_name: |
| 149 | + fail(package + ": no readme") |
| 150 | + readme = "" |
| 151 | + else: |
| 152 | + try: |
| 153 | + readme = z.read(readme_name).decode("utf-8", "replace") |
| 154 | + except KeyError: |
| 155 | + fail("%s: readme %s not in the package" % (package, readme_name)) |
| 156 | + readme = "" |
| 157 | + urls = [] |
| 158 | + for source, body in (("projectUrl", project_url), ("repository", repo_url), ("releaseNotes", notes), ("README", readme)): |
| 159 | + for u in URL.findall(body): |
| 160 | + u = strip_trailing(u) |
| 161 | + if LOCAL.match(u): |
| 162 | + continue |
| 163 | + if u not in [x[1] for x in urls]: |
| 164 | + urls.append((source, u)) |
| 165 | + print(" %d urls" % len(urls)) |
| 166 | + for source, u in urls: |
| 167 | + s = status(u) |
| 168 | + ok = isinstance(s, int) and 200 <= s < 300 |
| 169 | + print(" %s %s (%s)" % ("ok " if ok else "FAIL", u, source)) |
| 170 | + if not ok: |
| 171 | + fail("%s: %s from %s -> %s" % (package, u, source, s)) |
| 172 | + |
| 173 | + |
| 174 | +def check_obsoletions(version, workdir): |
| 175 | + print("== obsolete members in the published core assembly") |
| 176 | + scratch = os.path.join(workdir, "obsoletions") |
| 177 | + os.makedirs(scratch, exist_ok=True) |
| 178 | + with open(os.path.join(scratch, "Obsoletions.csproj"), "w", encoding="utf-8") as f: |
| 179 | + f.write(PROJECT.format(version=version)) |
| 180 | + with open(os.path.join(scratch, "Program.cs"), "w", encoding="utf-8") as f: |
| 181 | + f.write(PROGRAM) |
| 182 | + with open(os.path.join(scratch, "nuget.config"), "w", encoding="utf-8") as f: |
| 183 | + f.write(NUGET_CONFIG) |
| 184 | + env = dict(os.environ, NUGET_PACKAGES=os.path.join(workdir, "packages"), DOTNET_CLI_TELEMETRY_OPTOUT="1", DOTNET_NOLOGO="1") |
| 185 | + run = subprocess.run(["dotnet", "run", "--project", scratch, "-c", "Release"], capture_output=True, text=True, env=env) |
| 186 | + if run.returncode != 0: |
| 187 | + fail("scratch project failed:\n" + run.stdout[-3000:] + run.stderr[-3000:]) |
| 188 | + return |
| 189 | + seen = {} |
| 190 | + for line in run.stdout.splitlines(): |
| 191 | + parts = line.strip().split("|") |
| 192 | + if parts[0] == "assembly": |
| 193 | + print(" " + parts[1]) |
| 194 | + if "PublicKeyToken=null" in parts[1]: |
| 195 | + fail("core assembly is not strong-named: " + parts[1]) |
| 196 | + continue |
| 197 | + if len(parts) != 4: |
| 198 | + continue |
| 199 | + member, diag_id, url_format, message = parts |
| 200 | + seen.setdefault(member, set()).add((diag_id, url_format)) |
| 201 | + for member, expected in OBSOLETE.items(): |
| 202 | + if member not in seen: |
| 203 | + fail(member + ": no ObsoleteAttribute found") |
| 204 | + continue |
| 205 | + for diag_id, url_format in seen[member]: |
| 206 | + if diag_id != expected: |
| 207 | + fail("%s: DiagnosticId %s, expected %s" % (member, diag_id, expected)) |
| 208 | + continue |
| 209 | + if not url_format: |
| 210 | + fail(member + ": no UrlFormat") |
| 211 | + continue |
| 212 | + page = url_format.format(diag_id) |
| 213 | + anchor = page.split("#", 1)[1] if "#" in page else "" |
| 214 | + try: |
| 215 | + code, html = fetch(page.split("#", 1)[0]) |
| 216 | + except urllib.error.HTTPError as e: |
| 217 | + fail("%s: %s -> %s" % (member, page, e.code)) |
| 218 | + continue |
| 219 | + has_anchor = anchor and ('id="%s"' % anchor) in html |
| 220 | + print(" %s %s -> %s %s" % ("ok " if has_anchor else "FAIL", member, page, "anchor present" if has_anchor else "anchor missing")) |
| 221 | + if not has_anchor: |
| 222 | + fail("%s: anchor %s not in %s" % (member, anchor, page)) |
| 223 | + |
| 224 | + |
| 225 | +def main(): |
| 226 | + args = [a for a in sys.argv[1:] if not a.startswith("--")] |
| 227 | + keep = "--keep" in sys.argv |
| 228 | + if len(args) != 1: |
| 229 | + print(__doc__) |
| 230 | + sys.exit(2) |
| 231 | + version = args[0] |
| 232 | + workdir = tempfile.mkdtemp(prefix="jsonrpc-links-") |
| 233 | + try: |
| 234 | + for package in PACKAGES: |
| 235 | + check_package(package, version, workdir) |
| 236 | + check_obsoletions(version, workdir) |
| 237 | + finally: |
| 238 | + if keep: |
| 239 | + print("kept " + workdir) |
| 240 | + else: |
| 241 | + shutil.rmtree(workdir, ignore_errors=True) |
| 242 | + if failures: |
| 243 | + print("\n%d failure(s)" % len(failures)) |
| 244 | + sys.exit(1) |
| 245 | + print("\nall package links and obsoletion anchors resolve for " + version) |
| 246 | + |
| 247 | + |
| 248 | +if __name__ == "__main__": |
| 249 | + main() |
0 commit comments