diff --git a/generateForge.py b/generateForge.py index b49690c..9c8cc49 100755 --- a/generateForge.py +++ b/generateForge.py @@ -134,7 +134,13 @@ def version_from_modernized_installer(installer: MojangVersion, version: ForgeVe mc_filter = load_mc_version_filter(mc_version) for upstream_lib in installer.libraries: - forge_lib = Library.parse_obj(upstream_lib.dict()) # "cast" MojangLibrary to Library + forge_lib = Library( + extract=upstream_lib.extract, + name=upstream_lib.name, + downloads=upstream_lib.downloads, + natives=upstream_lib.natives, + rules=upstream_lib.rules, + ) # "cast" MojangLibrary to Library if forge_lib.name.is_lwjgl() or forge_lib.name.is_log4j() or should_ignore_artifact(mc_filter, forge_lib.name): continue @@ -204,7 +210,13 @@ def version_from_build_system_installer(installer: MojangVersion, profile: Forge v.maven_files.append(installer_lib) for upstream_lib in profile.libraries: - forge_lib = Library.parse_obj(upstream_lib.dict()) + forge_lib = Library( + extract=upstream_lib.extract, + name=upstream_lib.name, + downloads=upstream_lib.downloads, + natives=upstream_lib.natives, + rules=upstream_lib.rules, + ) if forge_lib.name.is_log4j(): continue @@ -223,7 +235,13 @@ def version_from_build_system_installer(installer: MojangVersion, profile: Forge v.libraries.append(wrapper_lib) for upstream_lib in installer.libraries: - forge_lib = Library.parse_obj(upstream_lib.dict()) + forge_lib = Library( + extract=upstream_lib.extract, + name=upstream_lib.name, + downloads=upstream_lib.downloads, + natives=upstream_lib.natives, + rules=upstream_lib.rules, + ) if forge_lib.name.is_log4j(): continue @@ -368,13 +386,12 @@ def main(): v.write(os.path.join(PMC_DIR, FORGE_COMPONENT, f"{v.version}.json")) - recommended_versions.sort() + recommended_versions.sort() + print('Recommended versions:', recommended_versions) - print('Recommended versions:', recommended_versions) - - package = MetaPackage(uid=FORGE_COMPONENT, name="Forge", project_url="https://www.minecraftforge.net/forum/") - package.recommended = recommended_versions - package.write(os.path.join(PMC_DIR, FORGE_COMPONENT, "package.json")) + package = MetaPackage(uid=FORGE_COMPONENT, name="Forge", project_url="https://www.minecraftforge.net/forum/") + package.recommended = recommended_versions + package.write(os.path.join(PMC_DIR, FORGE_COMPONENT, "package.json")) if __name__ == '__main__': diff --git a/generateNeoforge.py b/generateNeoforge.py index e21985d..6a1e3c7 100644 --- a/generateNeoforge.py +++ b/generateNeoforge.py @@ -1,18 +1,17 @@ -from meta.common.json import dump, dumps, load, loads +from datetime import datetime + +from meta.common.json import dumps, load import os import sys -from meta.common import ensure_component_dir, polymc_path, upstream_path, static_path +from meta.common import ensure_component_dir, polymc_path, upstream_path, serialize_datetime from meta.common.mojang import MINECRAFT_COMPONENT from meta.common.neoforge import NEOFORGE_COMPONENT, INSTALLER_MANIFEST_DIR, VERSION_MANIFEST_DIR, DERIVED_INDEX_FILE -from meta.model import MetaVersion, Dependency, Library, GradleSpecifier, MojangLibraryDownloads, MojangArtifact, \ - MetaPackage from meta.model.mojang import MojangVersion from meta.model.neoforge import NeoForgeEntry, NeoForgeInstallerProfile PMC_DIR = polymc_path() UPSTREAM_DIR = upstream_path() -STATIC_DIR = static_path() ensure_component_dir(NEOFORGE_COMPONENT) @@ -21,61 +20,92 @@ def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def version_from_installer(installer: MojangVersion, profile: NeoForgeInstallerProfile, - entry: NeoForgeEntry) -> MetaVersion: - v = MetaVersion(name="NeoForge", version=entry.sane_version(), uid=NEOFORGE_COMPONENT) - v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=entry.mc_version)] - v.main_class = "io.github.zekerzhayard.forgewrapper.installer.Main" - - # FIXME: Add the size and hash here - v.maven_files = [] - - # load the locally cached installer file info and use it to add the installer entry in the json - installer_lib = Library( - name=GradleSpecifier("net.neoforged", "forge" if entry.mc_version == "1.20.1" else "neoforge", - entry.version, "installer")) - installer_lib.downloads = MojangLibraryDownloads() - - installer_lib.downloads.artifact = MojangArtifact( - url=entry.installer_url(), - sha1=entry.installer_sha1, - size=entry.installer_size) - v.maven_files.append(installer_lib) +def _lib_dict(name_str, downloads=None, extract=None, natives=None, rules=None): + d = {"name": name_str} + if downloads: + d["downloads"] = downloads + if extract: + d["extract"] = extract + if natives: + d["natives"] = natives + if rules: + d["rules"] = rules + return d - for upstream_lib in profile.libraries: - forge_lib = Library.parse_obj(upstream_lib.dict()) - if forge_lib.name.is_log4j(): - continue - v.maven_files.append(forge_lib) +def _artifact_dict(url, sha1=None, size=None, path=None): + d = {"url": url} + if sha1: + d["sha1"] = sha1 + if size is not None: + d["size"] = size + if path: + d["path"] = path + return d - v.libraries = [] - wrapper_lib = Library(name=GradleSpecifier("io.github.zekerzhayard", "ForgeWrapper", "mmc7")) - wrapper_lib.downloads = MojangLibraryDownloads() - wrapper_lib.downloads.artifact = MojangArtifact( - url="https://github.com/MultiMC/ForgeWrapper/releases/download/mmc7/ForgeWrapper-mmc7.jar", - sha1="0c99747406998c933be78a368dfd8386949d1935", - size=29346 - ) - v.libraries.append(wrapper_lib) +def _lib_from_upstream(upstream_lib): + name_str = str(upstream_lib.name) + downloads = None + if upstream_lib.downloads and upstream_lib.downloads.artifact: + a = upstream_lib.downloads.artifact + downloads = {"artifact": _artifact_dict(url=a.url, sha1=a.sha1, size=a.size, path=a.path)} + return _lib_dict(name_str, downloads=downloads) - for upstream_lib in installer.libraries: - forge_lib = Library.parse_obj(upstream_lib.dict()) - if forge_lib.name.is_log4j(): + +def version_from_installer(installer: MojangVersion, profile: NeoForgeInstallerProfile, + entry: NeoForgeEntry): + loader = "forge" if entry.mc_version == "1.20.1" else "neoforge" + installer_name = f"net.neoforged:{loader}:{entry.version}:installer" + installer_url = entry.installer_url() + + maven_files = [] + maven_files.append(_lib_dict( + installer_name, + downloads={"artifact": _artifact_dict(url=installer_url, sha1=entry.installer_sha1, size=entry.installer_size)} + )) + + for upstream_lib in profile.libraries: + if upstream_lib.name.is_log4j(): continue + maven_files.append(_lib_from_upstream(upstream_lib)) + + libraries = [] + libraries.append(_lib_dict( + "io.github.zekerzhayard:ForgeWrapper:mmc7", + downloads={"artifact": _artifact_dict( + url="https://github.com/MultiMC/ForgeWrapper/releases/download/mmc7/ForgeWrapper-mmc7.jar", + sha1="0c99747406998c933be78a368dfd8386949d1935", + size=29346 + )} + )) - v.libraries.append(forge_lib) + for upstream_lib in installer.libraries: + if upstream_lib.name.is_log4j(): + continue + libraries.append(_lib_from_upstream(upstream_lib)) - v.release_time = installer.release_time - v.order = 5 mc_args = "--username ${auth_player_name} --version ${version_name} --gameDir ${game_directory} " \ "--assetsDir ${assets_root} --assetIndex ${assets_index_name} --uuid ${auth_uuid} " \ "--accessToken ${auth_access_token} --userType ${user_type} --versionType ${version_type}" for arg in installer.arguments.game: mc_args += f" {arg}" - v.minecraft_arguments = mc_args - return v + + release_time = serialize_datetime(installer.release_time) if isinstance(installer.release_time, datetime) else installer.release_time + + return { + "formatVersion": 1, + "name": "NeoForge", + "version": entry.sane_version(), + "uid": NEOFORGE_COMPONENT, + "requires": [{"uid": MINECRAFT_COMPONENT, "equals": entry.mc_version}], + "mainClass": "io.github.zekerzhayard.forgewrapper.installer.Main", + "mavenFiles": maven_files, + "libraries": libraries, + "order": 5, + "releaseTime": release_time, + "minecraftArguments": mc_args, + } def main(): @@ -105,14 +135,16 @@ def main(): installer = MojangVersion.parse_file(installer_version_filepath) v = version_from_installer(installer, profile, entry) - v.write(os.path.join(PMC_DIR, NEOFORGE_COMPONENT, f"{v.version}.json")) + with open(os.path.join(PMC_DIR, NEOFORGE_COMPONENT, f"{v['version']}.json"), 'w') as f: + f.write(dumps(v, indent=4, sort_keys=True)) - recommended_versions.sort() - print('Recommended versions:', recommended_versions) + recommended_versions.sort() + print('Recommended versions:', recommended_versions) - package = MetaPackage(uid=NEOFORGE_COMPONENT, name="NeoForge", project_url="https://neoforged.net/") - package.recommended = recommended_versions - package.write(os.path.join(PMC_DIR, NEOFORGE_COMPONENT, "package.json")) + from meta.model import MetaPackage + package = MetaPackage(uid=NEOFORGE_COMPONENT, name="NeoForge", project_url="https://neoforged.net/") + package.recommended = recommended_versions + package.write(os.path.join(PMC_DIR, NEOFORGE_COMPONENT, "package.json")) if __name__ == '__main__': diff --git a/meta/common/http.py b/meta/common/http.py index c057e4b..57f9960 100644 --- a/meta/common/http.py +++ b/meta/common/http.py @@ -2,5 +2,5 @@ def download_binary_file(sess, path, url): with open(path, 'wb') as f: r = sess.get(url) r.raise_for_status() - for chunk in r.iter_content(chunk_size=128): + for chunk in r.iter_content(chunk_size=8192): f.write(chunk) diff --git a/meta/model/__init__.py b/meta/model/__init__.py index c1ad0f7..7bcd88b 100644 --- a/meta/model/__init__.py +++ b/meta/model/__init__.py @@ -11,6 +11,24 @@ META_FORMAT_VERSION = 1 +# dict-friendly type conversion but without pydantic nonsense +def _fast_convert(value): + tp = type(value) + if tp is list: + return [_fast_convert(v) for v in value] + if tp is dict: + return {k: _fast_convert(v) for k, v in value.items()} + if tp is str or tp is int or tp is float or tp is bool or value is None: + return value + if tp is GradleSpecifier: + return str(value) + if tp is datetime: + return serialize_datetime(value) + if isinstance(value, MetaBase): + return value.fast_to_dict() + return value + + class GradleSpecifier: """ A gradle specifier - a maven coordinate. Like one of these: @@ -117,9 +135,23 @@ def json(self, **kwargs: Any) -> str: return super(MetaBase, self).json(exclude_none=True, sort_keys=True, by_alias=True, indent=4, **kwargs) + # faster dict conversion without pydantic nonsense + # this primarily is done to avoid recursion + def fast_to_dict(self) -> Dict[str, Any]: + if "__root__" in self.__fields__: + return _fast_convert(self.__root__) + result = {} + for field_name, field_obj in self.__fields__.items(): + alias = field_obj.alias or field_name + value = getattr(self, field_name) + if value is None: + continue + result[alias] = _fast_convert(value) + return result + def write(self, file_path): with open(file_path, "w") as f: - f.write(self.json()) + f.write(dumps(self.fast_to_dict(), indent=4, sort_keys=True)) def merge(self, other): """ diff --git a/updateFabric.py b/updateFabric.py index cc7eb2a..a8c11e7 100755 --- a/updateFabric.py +++ b/updateFabric.py @@ -60,7 +60,7 @@ def get_binary_file(path, url, sess): with open(path, 'wb') as f: r = sess.get(url) r.raise_for_status() - for chunk in r.iter_content(chunk_size=128): + for chunk in r.iter_content(chunk_size=8192): f.write(chunk) diff --git a/updateForge.py b/updateForge.py index 1881226..8e29524 100755 --- a/updateForge.py +++ b/updateForge.py @@ -274,7 +274,7 @@ def main(): rfile = sess.get(version.url(), stream=True) rfile.raise_for_status() with open(jar_path, 'wb') as f: - for chunk in rfile.iter_content(chunk_size=128): + for chunk in rfile.iter_content(chunk_size=8192): f.write(chunk) eprint("Processing %s" % version.url()) @@ -343,7 +343,7 @@ def main(): rfile = sess.get(version.url(), stream=True) rfile.raise_for_status() with open(jar_path, 'wb') as f: - for chunk in rfile.iter_content(chunk_size=128): + for chunk in rfile.iter_content(chunk_size=8192): f.write(chunk) # find the latest timestamp in the zip file tstamp = datetime.fromtimestamp(0) diff --git a/updateNeoforge.py b/updateNeoforge.py index cd3831e..4b49eb0 100644 --- a/updateNeoforge.py +++ b/updateNeoforge.py @@ -110,7 +110,7 @@ def main(): installer_file = sess.get(entry.installer_url(), stream=True) installer_file.raise_for_status() with open(jar_path, 'wb') as f: - for chunk in installer_file.iter_content(chunk_size=128): + for chunk in installer_file.iter_content(chunk_size=8192): f.write(chunk) if not os.path.isfile(jar_path): diff --git a/updateQuilt.py b/updateQuilt.py index 021dcce..4643d95 100755 --- a/updateQuilt.py +++ b/updateQuilt.py @@ -62,7 +62,7 @@ def get_binary_file(path, url, sess): with open(path, 'wb') as f: r = sess.get(url) r.raise_for_status() - for chunk in r.iter_content(chunk_size=128): + for chunk in r.iter_content(chunk_size=8192): f.write(chunk)