refactor: Experimental optimizations

generateForge.py

@@ -134,7 +134,13 @@ def version_from_modernized_installer(installer: MojangVersion, version: ForgeVe
 
     mc_filter = load_mc_version_filter(mc_version)
     for upstream_lib in installer.libraries:
-        forge_lib = Library.parse_obj(upstream_lib.dict())  # "cast" MojangLibrary to Library
+        forge_lib = Library(
+            extract=upstream_lib.extract,
+            name=upstream_lib.name,
+            downloads=upstream_lib.downloads,
+            natives=upstream_lib.natives,
+            rules=upstream_lib.rules,
+        )  # "cast" MojangLibrary to Library
         if forge_lib.name.is_lwjgl() or forge_lib.name.is_log4j() or should_ignore_artifact(mc_filter, forge_lib.name):
             continue
 
@@ -204,7 +210,13 @@ def version_from_build_system_installer(installer: MojangVersion, profile: Forge
     v.maven_files.append(installer_lib)
 
     for upstream_lib in profile.libraries:
-        forge_lib = Library.parse_obj(upstream_lib.dict())
+        forge_lib = Library(
+            extract=upstream_lib.extract,
+            name=upstream_lib.name,
+            downloads=upstream_lib.downloads,
+            natives=upstream_lib.natives,
+            rules=upstream_lib.rules,
+        )
         if forge_lib.name.is_log4j():
             continue
 
@@ -223,7 +235,13 @@ def version_from_build_system_installer(installer: MojangVersion, profile: Forge
     v.libraries.append(wrapper_lib)
 
     for upstream_lib in installer.libraries:
-        forge_lib = Library.parse_obj(upstream_lib.dict())
+        forge_lib = Library(
+            extract=upstream_lib.extract,
+            name=upstream_lib.name,
+            downloads=upstream_lib.downloads,
+            natives=upstream_lib.natives,
+            rules=upstream_lib.rules,
+        )
         if forge_lib.name.is_log4j():
             continue
 
@@ -368,13 +386,12 @@ def main():
 
         v.write(os.path.join(PMC_DIR, FORGE_COMPONENT, f"{v.version}.json"))
 
-        recommended_versions.sort()
+    recommended_versions.sort()
+    print('Recommended versions:', recommended_versions)
 
-        print('Recommended versions:', recommended_versions)
-
-        package = MetaPackage(uid=FORGE_COMPONENT, name="Forge", project_url="https://www.minecraftforge.net/forum/")
-        package.recommended = recommended_versions
-        package.write(os.path.join(PMC_DIR, FORGE_COMPONENT, "package.json"))
+    package = MetaPackage(uid=FORGE_COMPONENT, name="Forge", project_url="https://www.minecraftforge.net/forum/")
+    package.recommended = recommended_versions
+    package.write(os.path.join(PMC_DIR, FORGE_COMPONENT, "package.json"))
 
 
 if __name__ == '__main__':

generateNeoforge.py

@@ -1,18 +1,17 @@
-from meta.common.json import dump, dumps, load, loads
+from datetime import datetime
+
+from meta.common.json import dumps, load
 import os
 import sys
 
-from meta.common import ensure_component_dir, polymc_path, upstream_path, static_path
+from meta.common import ensure_component_dir, polymc_path, upstream_path, serialize_datetime
 from meta.common.mojang import MINECRAFT_COMPONENT
 from meta.common.neoforge import NEOFORGE_COMPONENT, INSTALLER_MANIFEST_DIR, VERSION_MANIFEST_DIR, DERIVED_INDEX_FILE
-from meta.model import MetaVersion, Dependency, Library, GradleSpecifier, MojangLibraryDownloads, MojangArtifact, \
-    MetaPackage
 from meta.model.mojang import MojangVersion
 from meta.model.neoforge import NeoForgeEntry, NeoForgeInstallerProfile
 
 PMC_DIR = polymc_path()
 UPSTREAM_DIR = upstream_path()
-STATIC_DIR = static_path()
 
 ensure_component_dir(NEOFORGE_COMPONENT)
 
@@ -21,61 +20,92 @@ def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
 
-def version_from_installer(installer: MojangVersion, profile: NeoForgeInstallerProfile,
-                           entry: NeoForgeEntry) -> MetaVersion:
-    v = MetaVersion(name="NeoForge", version=entry.sane_version(), uid=NEOFORGE_COMPONENT)
-    v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=entry.mc_version)]
-    v.main_class = "io.github.zekerzhayard.forgewrapper.installer.Main"
-
-    # FIXME: Add the size and hash here
-    v.maven_files = []
-
-    # load the locally cached installer file info and use it to add the installer entry in the json
-    installer_lib = Library(
-        name=GradleSpecifier("net.neoforged", "forge" if entry.mc_version == "1.20.1" else "neoforge",
-                             entry.version, "installer"))
-    installer_lib.downloads = MojangLibraryDownloads()
-
-    installer_lib.downloads.artifact = MojangArtifact(
-        url=entry.installer_url(),
-        sha1=entry.installer_sha1,
-        size=entry.installer_size)
-    v.maven_files.append(installer_lib)
+def _lib_dict(name_str, downloads=None, extract=None, natives=None, rules=None):
+    d = {"name": name_str}
+    if downloads:
+        d["downloads"] = downloads
+    if extract:
+        d["extract"] = extract
+    if natives:
+        d["natives"] = natives
+    if rules:
+        d["rules"] = rules
+    return d
 
-    for upstream_lib in profile.libraries:
-        forge_lib = Library.parse_obj(upstream_lib.dict())
-        if forge_lib.name.is_log4j():
-            continue
 
-        v.maven_files.append(forge_lib)
+def _artifact_dict(url, sha1=None, size=None, path=None):
+    d = {"url": url}
+    if sha1:
+        d["sha1"] = sha1
+    if size is not None:
+        d["size"] = size
+    if path:
+        d["path"] = path
+    return d
 
-    v.libraries = []
 
-    wrapper_lib = Library(name=GradleSpecifier("io.github.zekerzhayard", "ForgeWrapper", "mmc7"))
-    wrapper_lib.downloads = MojangLibraryDownloads()
-    wrapper_lib.downloads.artifact = MojangArtifact(
-        url="https://github.com/MultiMC/ForgeWrapper/releases/download/mmc7/ForgeWrapper-mmc7.jar",
-        sha1="0c99747406998c933be78a368dfd8386949d1935",
-        size=29346
-    )
-    v.libraries.append(wrapper_lib)
+def _lib_from_upstream(upstream_lib):
+    name_str = str(upstream_lib.name)
+    downloads = None
+    if upstream_lib.downloads and upstream_lib.downloads.artifact:
+        a = upstream_lib.downloads.artifact
+        downloads = {"artifact": _artifact_dict(url=a.url, sha1=a.sha1, size=a.size, path=a.path)}
+    return _lib_dict(name_str, downloads=downloads)
 
-    for upstream_lib in installer.libraries:
-        forge_lib = Library.parse_obj(upstream_lib.dict())
-        if forge_lib.name.is_log4j():
+
+def version_from_installer(installer: MojangVersion, profile: NeoForgeInstallerProfile,
+                           entry: NeoForgeEntry):
+    loader = "forge" if entry.mc_version == "1.20.1" else "neoforge"
+    installer_name = f"net.neoforged:{loader}:{entry.version}:installer"
+    installer_url = entry.installer_url()
+
+    maven_files = []
+    maven_files.append(_lib_dict(
+        installer_name,
+        downloads={"artifact": _artifact_dict(url=installer_url, sha1=entry.installer_sha1, size=entry.installer_size)}
+    ))
+
+    for upstream_lib in profile.libraries:
+        if upstream_lib.name.is_log4j():
             continue
+        maven_files.append(_lib_from_upstream(upstream_lib))
+
+    libraries = []
+    libraries.append(_lib_dict(
+        "io.github.zekerzhayard:ForgeWrapper:mmc7",
+        downloads={"artifact": _artifact_dict(
+            url="https://github.com/MultiMC/ForgeWrapper/releases/download/mmc7/ForgeWrapper-mmc7.jar",
+            sha1="0c99747406998c933be78a368dfd8386949d1935",
+            size=29346
+        )}
+    ))
 
-        v.libraries.append(forge_lib)
+    for upstream_lib in installer.libraries:
+        if upstream_lib.name.is_log4j():
+            continue
+        libraries.append(_lib_from_upstream(upstream_lib))
 
-    v.release_time = installer.release_time
-    v.order = 5
     mc_args = "--username ${auth_player_name} --version ${version_name} --gameDir ${game_directory} " \
               "--assetsDir ${assets_root} --assetIndex ${assets_index_name} --uuid ${auth_uuid} " \
               "--accessToken ${auth_access_token} --userType ${user_type} --versionType ${version_type}"
     for arg in installer.arguments.game:
         mc_args += f" {arg}"
-    v.minecraft_arguments = mc_args
-    return v
+
+    release_time = serialize_datetime(installer.release_time) if isinstance(installer.release_time, datetime) else installer.release_time
+
+    return {
+        "formatVersion": 1,
+        "name": "NeoForge",
+        "version": entry.sane_version(),
+        "uid": NEOFORGE_COMPONENT,
+        "requires": [{"uid": MINECRAFT_COMPONENT, "equals": entry.mc_version}],
+        "mainClass": "io.github.zekerzhayard.forgewrapper.installer.Main",
+        "mavenFiles": maven_files,
+        "libraries": libraries,
+        "order": 5,
+        "releaseTime": release_time,
+        "minecraftArguments": mc_args,
+    }
 
 
 def main():
@@ -105,14 +135,16 @@ def main():
         installer = MojangVersion.parse_file(installer_version_filepath)
         v = version_from_installer(installer, profile, entry)
 
-        v.write(os.path.join(PMC_DIR, NEOFORGE_COMPONENT, f"{v.version}.json"))
+        with open(os.path.join(PMC_DIR, NEOFORGE_COMPONENT, f"{v['version']}.json"), 'w') as f:
+            f.write(dumps(v, indent=4, sort_keys=True))
 
-        recommended_versions.sort()
-        print('Recommended versions:', recommended_versions)
+    recommended_versions.sort()
+    print('Recommended versions:', recommended_versions)
 
-        package = MetaPackage(uid=NEOFORGE_COMPONENT, name="NeoForge", project_url="https://neoforged.net/")
-        package.recommended = recommended_versions
-        package.write(os.path.join(PMC_DIR, NEOFORGE_COMPONENT, "package.json"))
+    from meta.model import MetaPackage
+    package = MetaPackage(uid=NEOFORGE_COMPONENT, name="NeoForge", project_url="https://neoforged.net/")
+    package.recommended = recommended_versions
+    package.write(os.path.join(PMC_DIR, NEOFORGE_COMPONENT, "package.json"))
 
 
 if __name__ == '__main__':

meta/common/http.py

@@ -2,5 +2,5 @@ def download_binary_file(sess, path, url):
     with open(path, 'wb') as f:
         r = sess.get(url)
         r.raise_for_status()
-        for chunk in r.iter_content(chunk_size=128):
+        for chunk in r.iter_content(chunk_size=8192):
             f.write(chunk)

meta/model/__init__.py

@@ -11,6 +11,24 @@
 META_FORMAT_VERSION = 1
 
 
+# dict-friendly type conversion but without pydantic nonsense
+def _fast_convert(value):
+    tp = type(value)
+    if tp is list:
+        return [_fast_convert(v) for v in value]
+    if tp is dict:
+        return {k: _fast_convert(v) for k, v in value.items()}
+    if tp is str or tp is int or tp is float or tp is bool or value is None:
+        return value
+    if tp is GradleSpecifier:
+        return str(value)
+    if tp is datetime:
+        return serialize_datetime(value)
+    if isinstance(value, MetaBase):
+        return value.fast_to_dict()
+    return value
+
+
 class GradleSpecifier:
     """
         A gradle specifier - a maven coordinate. Like one of these:
@@ -117,9 +135,23 @@ def json(self, **kwargs: Any) -> str:
 
         return super(MetaBase, self).json(exclude_none=True, sort_keys=True, by_alias=True, indent=4, **kwargs)
 
+    # faster dict conversion without pydantic nonsense
+    # this primarily is done to avoid recursion
+    def fast_to_dict(self) -> Dict[str, Any]:
+        if "__root__" in self.__fields__:
+            return _fast_convert(self.__root__)
+        result = {}
+        for field_name, field_obj in self.__fields__.items():
+            alias = field_obj.alias or field_name
+            value = getattr(self, field_name)
+            if value is None:
+                continue
+            result[alias] = _fast_convert(value)
+        return result
+
     def write(self, file_path):
         with open(file_path, "w") as f:
-            f.write(self.json())
+            f.write(dumps(self.fast_to_dict(), indent=4, sort_keys=True))
 
     def merge(self, other):
         """

updateFabric.py

@@ -60,7 +60,7 @@ def get_binary_file(path, url, sess):
     with open(path, 'wb') as f:
         r = sess.get(url)
         r.raise_for_status()
-        for chunk in r.iter_content(chunk_size=128):
+        for chunk in r.iter_content(chunk_size=8192):
             f.write(chunk)
 
 

updateForge.py

@@ -274,7 +274,7 @@ def main():
                     rfile = sess.get(version.url(), stream=True)
                     rfile.raise_for_status()
                     with open(jar_path, 'wb') as f:
-                        for chunk in rfile.iter_content(chunk_size=128):
+                        for chunk in rfile.iter_content(chunk_size=8192):
                             f.write(chunk)
 
             eprint("Processing %s" % version.url())
@@ -343,7 +343,7 @@ def main():
                     rfile = sess.get(version.url(), stream=True)
                     rfile.raise_for_status()
                     with open(jar_path, 'wb') as f:
-                        for chunk in rfile.iter_content(chunk_size=128):
+                        for chunk in rfile.iter_content(chunk_size=8192):
                             f.write(chunk)
                 # find the latest timestamp in the zip file
                 tstamp = datetime.fromtimestamp(0)

updateNeoforge.py

@@ -110,7 +110,7 @@ def main():
             installer_file = sess.get(entry.installer_url(), stream=True)
             installer_file.raise_for_status()
             with open(jar_path, 'wb') as f:
-                for chunk in installer_file.iter_content(chunk_size=128):
+                for chunk in installer_file.iter_content(chunk_size=8192):
                     f.write(chunk)
 
         if not os.path.isfile(jar_path):

updateQuilt.py

@@ -62,7 +62,7 @@ def get_binary_file(path, url, sess):
     with open(path, 'wb') as f:
         r = sess.get(url)
         r.raise_for_status()
-        for chunk in r.iter_content(chunk_size=128):
+        for chunk in r.iter_content(chunk_size=8192):
             f.write(chunk)