From 347ccbd30e7009fc6aa623a89841db7f6f39c43c Mon Sep 17 00:00:00 2001 From: Takahiro Ueda Date: Sun, 24 Nov 2019 17:59:06 +0900 Subject: [PATCH 1/4] feat: add #UsePackage The new preprocessor instruction #UsePackage loads the specified package installed in "~/.form/". If the package is unavailable, the command will try to download and install it. Decent Unix shell and a set of commands (including Curl) are assumed to be available via the system() function. Example: #usepackage benruijl/forcer@v1.0.0 --- sources/Makefile.am | 1 + sources/declare.h | 2 + sources/package.cc | 435 ++++++++++++++++++++++++++++++++++++++++++++ sources/pre.c | 1 + 4 files changed, 439 insertions(+) create mode 100644 sources/package.cc diff --git a/sources/Makefile.am b/sources/Makefile.am index b0cd895d..226ae85c 100644 --- a/sources/Makefile.am +++ b/sources/Makefile.am @@ -35,6 +35,7 @@ SRCBASE = \ notation.c \ opera.c \ optimize.cc \ + package.cc \ pattern.c \ poly.cc \ poly.h \ diff --git a/sources/declare.h b/sources/declare.h index 5ecc5f31..b86be7c0 100644 --- a/sources/declare.h +++ b/sources/declare.h @@ -1658,6 +1658,8 @@ extern int AddToScratch(FILEHANDLE *,POSITION *,UBYTE *,POSITION *,int); extern int DoPreAppendPath(UBYTE *); extern int DoPrePrependPath(UBYTE *); +extern int DoPreUsePackage(UBYTE *); + extern int DoSwitch(PHEAD WORD *, WORD *); extern int DoEndSwitch(PHEAD WORD *, WORD *); extern SWITCHTABLE *FindCase(WORD, WORD); diff --git a/sources/package.cc b/sources/package.cc new file mode 100644 index 00000000..0452f84d --- /dev/null +++ b/sources/package.cc @@ -0,0 +1,435 @@ +/** @file package.cc + * + * The FORM package manager. + */ + +/* #[ License : */ +/* + * Copyright (C) 1984-2017 J.A.M. Vermaseren + * When using this file you are requested to refer to the publication + * J.A.M.Vermaseren "New features of FORM" math-ph/0010025 + * This is considered a matter of courtesy as the development was paid + * for by FOM the Dutch physics granting agency and we would like to + * be able to track its scientific use to convince FOM of its value + * for the community. + * + * This file is part of FORM. + * + * FORM is free software: you can redistribute it and/or modify it under the + * terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * FORM is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along + * with FORM. If not, see . + */ +/* #] License : */ + +#ifdef HAVE_CONFIG_H +#ifndef CONFIG_H_INCLUDED +#define CONFIG_H_INCLUDED +#include +#endif +#endif + +#include +#include +#include +#include +#include + +extern "C" { +#include "form3.h" +} + +namespace { + +/** + * Returns a copy of the given string with removing whitespace both at the + * beginning and the end. + */ +std::string Trim(const std::string& s) { + std::size_t i = 0; + std::size_t j = 0; + std::size_t k; + k = s.find_first_not_of(" \t"); + if (k != std::string::npos) { + i = k; + std::size_t k = s.find_last_not_of(" \t"); + if (k != std::string::npos) { + j = k + 1; + } + } + return s.substr(i, j - i); +} + +/** + * Checks if the file with the given name exists. + */ +bool FileExists(const std::string& filename) { + return std::ifstream(filename.c_str()).is_open(); +} + +/** + * Returns the path to the current user's home directory. + */ +std::string GetHomeDirectory() { + // TODO: more portability + const char* home = std::getenv("HOME"); + if (home) return home; + + home = std::getenv("HOMEPATH"); + if (home) return home; + + return ""; +} + +//@{ + +/** + * Joins two or more path components. + */ +std::string JoinPath(const std::string& path1, const std::string& path2) { + // Make copies. + std::string s1 = path1; + std::string s2 = path2; + // Canonicalize the paths in such a way that: + // (1) both path1 and path2 don't have the separator at the end. + // (2) path2 doesn't have the separator at the beginning. + if (!s1.empty()) { + if (s1[s1.length() - 1] == SEPARATOR || + s1[s1.length() - 1] == ALTSEPARATOR) { + s1.erase(s1.length() - 1); + } + } + if (!s2.empty()) { + if (s2[0] == SEPARATOR || s2[0] == ALTSEPARATOR) { + s2.erase(0); + } + } + if (!s2.empty()) { + if (s2[s2.length() - 1] == SEPARATOR || + s2[s2.length() - 1] == ALTSEPARATOR) { + s2.erase(s2.length() - 1); + } + } + // Join the two paths. + return s1 + SEPARATOR + s2; +} + +std::string JoinPath(const std::string& path1, const std::string& path2, + const std::string& path3) { + return JoinPath(path1, JoinPath(path2, path3)); +} + +std::string JoinPath(const std::string& path1, const std::string& path2, + const std::string& path3, const std::string& path4) { + return JoinPath(path1, JoinPath(path2, path3, path4)); +} + +std::string JoinPath(const std::string& path1, const std::string& path2, + const std::string& path3, const std::string& path4, + const std::string& path5) { + return JoinPath(path1, JoinPath(path2, path3, path4, path5)); +} + +std::string JoinPath(const std::string& path1, const std::string& path2, + const std::string& path3, const std::string& path4, + const std::string& path5, const std::string& path6) { + return JoinPath(path1, JoinPath(path2, path3, path4, path5, path6)); +} + +//@} + +/** + * Escapes the given path name. + */ +std::string EscapePath(const std::string& path) { + std::string s = path; + std::replace(s.begin(), s.end(), SEPARATOR, '_'); + std::replace(s.begin(), s.end(), ALTSEPARATOR, '_'); + std::replace(s.begin(), s.end(), ':', '_'); + std::replace(s.begin(), s.end(), '@', '_'); + std::replace(s.begin(), s.end(), ' ', '_'); + return s; +} + +/** + * Checks if the given file starts with "404" (indicating "File Not Found" on + * GitHub). + */ +bool Is404(const std::string& filename) { + char buf[3]; + std::ifstream file; + file.open(filename.c_str(), std::ios::in | std::ios::binary); + file.read(buf, 3); + return file && buf[0] == '4' && buf[1] == '0' && buf[2] == '4'; +} + +/** + * Deploys a package. + */ +int DeployPackage(const std::string& path, const std::string& url) { + // NOTE: we assume decent UNIX systems. +#ifdef UNIX + std::string cmd; + int err; + + // Create a temporary working directory. + + std::string pid = (const char*)GetPreVar((UBYTE*)"PID_", WITHERROR); + + std::string tmp_dir = "xformpkg" + pid; + + cmd = "mkdir -p " + tmp_dir; + err = DoSystem((UBYTE*)cmd.c_str()); + if (err) return err; + + // Download the package into the temporary directory. + + std::string tmp_name = JoinPath(tmp_dir, EscapePath(url)); + + cmd = "curl -L -o " + tmp_name + " " + url + " 2>&1"; + err = DoSystem((UBYTE*)cmd.c_str()); + if (err) return err; + + if (Is404(tmp_name)) { + return 404; + } + + // Uncompress the downloaded package. + + cmd = "cd " + tmp_dir + " && tar xfz *.tar.gz"; + err = DoSystem((UBYTE*)cmd.c_str()); + if (err) return err; + + // Relocate the package to the destination path. + + cmd = "mkdir -p $(dirname " + path + ")"; + err = DoSystem((UBYTE*)cmd.c_str()); + if (err) return err; + + cmd = "mv $(ls -d " + tmp_dir + "/*/) " + path; + err = DoSystem((UBYTE*)cmd.c_str()); + if (err) return err; + + // Create the ".complete" file. + + cmd = "touch " + JoinPath(path, ".complete"); + err = DoSystem((UBYTE*)cmd.c_str()); + if (err) return err; + + // Delete the working directory. + + cmd = "rm -rf " + tmp_dir; + err = DoSystem((UBYTE*)cmd.c_str()); + if (err) return err; + + // Congratulations! The package installation succeeded. + + return 0; +#else + MesPrint("Package installer not implemented on this system"); + return -1; +#endif +} + +/** + * Returns the package installation path. + */ +std::string GetPackagePath(const std::string& repo, const std::string& group, + const std::string& name, + const std::string& version) { + return JoinPath(GetHomeDirectory(), ".form", repo, group, name, version); +} + +/** + * Returns the package URL. + */ +std::string GetPackageURL(const std::string& repo, const std::string& group, + const std::string& name, const std::string& version) { + if (repo == "github.com") { + return "https://github.com/" + group + "/" + name + "/archive/" + version + + ".tar.gz"; + } else if (repo == "gitlab.com") { + return "https://gitlab.com/" + group + "/" + name + "/-/archive/" + + version + "/" + name + "-" + version + ".tar.gz"; + } else if (repo == "bitbucket.org") { + return "https://bitbucket.org/" + group + "/" + name + "/get/" + version + + ".tar.gz"; + } else { + return "https://" + repo + "/" + group + "/" + name + "/" + version + + ".tar.gz"; + } +} + +/** + * Ensures that the specified package is deployed. + */ +int EnsurePackage(const std::string& repo, const std::string& group, + const std::string& name, const std::string& version) { + std::string package_path = GetPackagePath(repo, group, name, version); + + std::string complete_file = JoinPath(package_path, ".complete"); + if (!FileExists(complete_file)) { + MesPrint("@Download package: %s", + (repo + "/" + group + "/" + name + "@" + version).c_str()); + + std::string package_url = GetPackageURL(repo, group, name, version); + int err = DeployPackage(package_path, package_url); + if (err == 404) { + MesPrint("@404: File Not Found"); + return -1; + } + if (err) return err; + + MesPrint("@Installed: %s", package_path.c_str()); + } + return 0; +} + +/** + * Splits a package name into its components. + */ +int SplitPackageNameComponents(const std::string& str, std::string* repo, + std::string* group, std::string* name, + std::string* version) { + std::string s = Trim(str); + bool rest_ignored = false; + + // Check if the string still has spaces. + if (!s.empty()) { + std::size_t i = s.find_first_of(" \t"); + if (i != std::string::npos) { + rest_ignored = true; + s.erase(i); + } + } + + // Find separators in the string. + std::size_t i = s.find_first_of("/"); + std::size_t j = s.find_first_of("/", i + 1); + std::size_t k = s.find_last_of("@"); + std::size_t l, m; + if (i == std::string::npos) { + goto illegal_format; + } + if (k == std::string::npos) { + goto illegal_format; + } + + // More checks. + l = s.find_first_of("@"); + m = s.find_first_of("/", k + 1); + if (l != k) { + goto illegal_format; + } + if (m != std::string::npos) { + goto illegal_format; + } + + // Split it. + if (j != std::string::npos && j < k) { + if (!(0 < i && i + 1 < j && j + 1 < k && k + 1 < s.size())) { + goto illegal_format; + } + *repo = s.substr(0, i); + *group = s.substr(i + 1, j - (i + 1)); + *name = s.substr(j + 1, k - (j + 1)); + *version = s.substr(k + 1); + } else { + if (!(0 < i && i + 1 < k && k + 1 < s.size())) { + goto illegal_format; + } + *repo = "github.com"; + *group = s.substr(0, i); + *name = s.substr(i + 1, k - (i + 1)); + *version = s.substr(k + 1); + } + + if (rest_ignored) { + MesPrint("&Text after whitespace in a package name is ignored"); + } + + return 0; + +illegal_format: + MesPrint("&Illegal package name: %s", s.c_str()); + return -1; +} + +} // unnamed namespace + +/** + * Loads the specified package. If the package is unavailable, it will be + * automatically downloaded and installed. + * + * Syntax: + * #UsePackage [-+] [/]/@ [: ] + */ +extern "C" int DoPreUsePackage(UBYTE* s) { + if (AP.PreSwitchModes[AP.PreSwitchLevel] != EXECUTINGPRESWITCH) return (0); + if (AP.PreIfStack[AP.PreIfLevel] != EXECUTINGIF) return (0); + + int err; + + int sign = 0; // "+" (1) or "-" (-1) or none (0). + + std::string line = Trim(std::string((const char*)s)); + std::string incl_args; + + // Check the optional sign. + if (!line.empty() && (line[0] == '+' || line[0] == '-')) { + sign = line[0] == '+' ? 1 : -1; + line.erase(0, 1); + line = Trim(line); + } + + // Check the optional include arguments. + { + std::size_t i = line.find_first_of(":"); + if (i != std::string::npos) { + incl_args = Trim(line.substr(i + 1)); + line = Trim(line.substr(0, i)); + } + } + + // First, we need to "Ensure Package". + + std::string repo; + std::string group; + std::string name; + std::string version; + + err = SplitPackageNameComponents(line, &repo, &group, &name, &version); + if (err) return err; + + err = EnsurePackage(repo, group, name, version); + if (err) return err; + + // Perform #PrependPath. + std::string include_path = + "\"" + GetPackagePath(repo, group, name, version) + "\""; + err = DoPreAppendPath((UBYTE*)include_path.c_str()); + if (err) return err; + + // Construct the arguments for #Include. By default, the main header file name + // is assumed to be "{name}.h". + if (incl_args.empty()) { + incl_args = name + ".h"; + } + if (sign != 0 && incl_args[0] != '+' && incl_args[0] != '-') { + incl_args = (sign > 0 ? "+ " : "- ") + incl_args; + } + + // Perform #Include. + err = DoInclude((UBYTE*)incl_args.c_str()); + if (err) return err; + + return 0; +} diff --git a/sources/pre.c b/sources/pre.c index c343210d..fb96c802 100644 --- a/sources/pre.c +++ b/sources/pre.c @@ -103,6 +103,7 @@ static KEYWORD precommands[] = { ,{"toexternal" , DoToExternal , 0, 0} ,{"undefine" , DoUndefine , 0, 0} ,{"usedictionary", DoPreUseDictionary,0,0} + ,{"usepackage" , DoPreUsePackage,0,0} ,{"write" , DoPreWrite , 0, 0} }; From c0e43b50eecf7b154b246e1ba4686a92f914c621 Mon Sep 17 00:00:00 2001 From: Takahiro Ueda Date: Sun, 24 Nov 2019 18:28:42 +0900 Subject: [PATCH 2/4] feat: add the -install option This option enables users to install a package in such a way that the package can be loaded by #UsePackage. Example: form -install benruijl/forcer@v1.0.0 --- sources/declare.h | 1 + sources/package.cc | 49 +++++++++++++++++++++++++++++++++++++++++++--- sources/startup.c | 34 ++++++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/sources/declare.h b/sources/declare.h index b86be7c0..329d75b7 100644 --- a/sources/declare.h +++ b/sources/declare.h @@ -1658,6 +1658,7 @@ extern int AddToScratch(FILEHANDLE *,POSITION *,UBYTE *,POSITION *,int); extern int DoPreAppendPath(UBYTE *); extern int DoPrePrependPath(UBYTE *); +extern int InstallPackage(UBYTE *); extern int DoPreUsePackage(UBYTE *); extern int DoSwitch(PHEAD WORD *, WORD *); diff --git a/sources/package.cc b/sources/package.cc index 0452f84d..f6e1d1ca 100644 --- a/sources/package.cc +++ b/sources/package.cc @@ -248,6 +248,20 @@ std::string GetPackagePath(const std::string& repo, const std::string& group, return JoinPath(GetHomeDirectory(), ".form", repo, group, name, version); } +/** + * Returns the package short name. + */ +std::string GetPackageDisplayName(const std::string& repo, + const std::string& group, + const std::string& name, + const std::string& version) { + if (repo == "github.com") { + return group + "/" + name + "@" + version; + } else { + return repo + "/" + group + "/" + name + "@" + version; + } +} + /** * Returns the package URL. */ @@ -272,13 +286,16 @@ std::string GetPackageURL(const std::string& repo, const std::string& group, * Ensures that the specified package is deployed. */ int EnsurePackage(const std::string& repo, const std::string& group, - const std::string& name, const std::string& version) { + const std::string& name, const std::string& version, + bool silent = true) { + std::string package_display_name = + GetPackageDisplayName(repo, group, name, version); std::string package_path = GetPackagePath(repo, group, name, version); std::string complete_file = JoinPath(package_path, ".complete"); + if (!FileExists(complete_file)) { - MesPrint("@Download package: %s", - (repo + "/" + group + "/" + name + "@" + version).c_str()); + MesPrint("@Download package: %s", package_display_name.c_str()); std::string package_url = GetPackageURL(repo, group, name, version); int err = DeployPackage(package_path, package_url); @@ -289,6 +306,8 @@ int EnsurePackage(const std::string& repo, const std::string& group, if (err) return err; MesPrint("@Installed: %s", package_path.c_str()); + } else if (!silent) { + MesPrint("@Already installed: %s", package_path.c_str()); } return 0; } @@ -365,6 +384,30 @@ int SplitPackageNameComponents(const std::string& str, std::string* repo, } // unnamed namespace +/** + * Installs the specified package. + * + * Syntax: + * form -install [/]/@ + */ +extern "C" int InstallPackage(UBYTE* s) { + int err; + + std::string repo; + std::string group; + std::string name; + std::string version; + + err = SplitPackageNameComponents(Trim(std::string((const char*)s)), &repo, + &group, &name, &version); + if (err) return err; + + err = EnsurePackage(repo, group, name, version, false); + if (err) return err; + + return 0; +} + /** * Loads the specified package. If the package is unavailable, it will be * automatically downloaded and installed. diff --git a/sources/startup.c b/sources/startup.c index b92ce1b9..cdd04ec2 100644 --- a/sources/startup.c +++ b/sources/startup.c @@ -257,10 +257,40 @@ int DoTail(int argc, UBYTE **argv) AM.FileOnlyFlag = 1; AM.LogType = 1; break; case 'h': /* For old systems: wait for key before exit */ AM.HoldFlag = 1; break; + case 'i': /* -i or -install */ + if ( s[1] == 'n' && s[2] == 's' && s[3] == 't' + && s[4] == 'a' && s[5] == 'l' && s[6] == 'l' + && s[7] == '\0' ) { + /* Install a package. */ + UBYTE *package; + TAKEPATH(package); + if ( package ) { + int err = InstallPackage(package); + if ( err ) return(err); + return(1); + } + else { +#ifdef WITHMPI + if ( PF.me == MASTER ) +#endif + printf("-install needs an argument\n"); + errorflag++; + } + } + else { #ifdef WITHINTERACTION - case 'i': /* Interactive session (not used yet) */ - AM.Interact = 1; break; + if ( s[1] == '\0' ) { + /* Interactive session (not used yet) */ + AM.Interact = 1; break; + } +#endif +#ifdef WITHMPI + if ( PF.me == MASTER ) #endif + printf("Illegal option: %s\n",s); + errorflag++; + } + break; case 'I': /* Next arg is dir for inc/prc/sub files */ TAKEPATH(AM.IncDir) break; case 'l': /* Make regular log file */ From b8bd4131edf0b56d6220108e6866b526754378e0 Mon Sep 17 00:00:00 2001 From: Takahiro Ueda Date: Sun, 24 Nov 2019 19:00:47 +0900 Subject: [PATCH 3/4] feat: package manager with wget The package manager now tries wget if curl is not available. --- sources/package.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sources/package.cc b/sources/package.cc index f6e1d1ca..3837c837 100644 --- a/sources/package.cc +++ b/sources/package.cc @@ -194,7 +194,13 @@ int DeployPackage(const std::string& path, const std::string& url) { std::string tmp_name = JoinPath(tmp_dir, EscapePath(url)); - cmd = "curl -L -o " + tmp_name + " " + url + " 2>&1"; + cmd = "command -v curl >/dev/null 2>&1"; + err = std::system(cmd.c_str()); + if (!err) { + cmd = "curl -L -o " + tmp_name + " " + url + " 2>&1"; + } else { + cmd = "wget -O " + tmp_name + " " + url + " 2>&1"; + } err = DoSystem((UBYTE*)cmd.c_str()); if (err) return err; From 881f3736cbd356bdce3d17fed676a788fbfd3e4c Mon Sep 17 00:00:00 2001 From: Takahiro Ueda Date: Mon, 25 Nov 2019 23:09:46 +0900 Subject: [PATCH 4/4] docs: -install and #usepackage --- doc/form.1 | 3 +++ doc/manual/prepro.tex | 52 ++++++++++++++++++++++++++++++++++++++++++ doc/manual/startup.tex | 3 +++ 3 files changed, 58 insertions(+) diff --git a/doc/form.1 b/doc/form.1 index 2a483d57..5bfeef46 100644 --- a/doc/form.1 +++ b/doc/form.1 @@ -42,6 +42,9 @@ Output only to log file. Further like \fB-L\fR or \fB-ll\fR. Wait for some key to be touched before finishing the run. Basically only for some old window based systems. .TP +.BR "-install" +Install the package specified by the next argument. +.TP .BR "-I" Next argument/option is the path of a directory for include, procedure and subroutine files. .TP diff --git a/doc/manual/prepro.tex b/doc/manual/prepro.tex index 232c78bb..59d34c60 100644 --- a/doc/manual/prepro.tex +++ b/doc/manual/prepro.tex @@ -2117,6 +2117,58 @@ \section{\#usedictionary} \noindent Starts using a dictionary for output translation. %--#] usedictionary : +%--#[ usepackage : + +\section{\#usepackage} +\label{preusepackage} + +\noindent Syntax: + +\#usepackage [$-+$] [site/]group/name@version [: [$-+$] filename [\# foldname]] + +\noindent See also include (\ref{preinclude}), + prependpath (\ref{preprependpath}). + +\noindent Loads the specified package\index{package!manager}. If the package is +not in the system, \FORM{} will try to download and install it, which can be +also done via the {\tt -install} option. This is implemented with the use of +the shell and {\tt curl} or {\tt wget} utility, therefore works only on Unix +systems. Packages will be installed into the {\tt .form} directory inside the +user's home directory. + +The package to be loaded by this instruction is specified in the following +format: +\begin{verbatim} + [site/]group/name@version +\end{verbatim} +Packages are expected to have been uploaded as public repositories on GitHub, +Bitbucket or GitLab. The optional {\tt site} component specifies the website +where the package exists: {\tt github.com}, {\tt bitbucket.org} or +{\tt gitlab.com}; the default value is {\tt github.com}. The {\tt group} +component specifies the group/owner account name and the {\tt name} component +is the repository name. The {\tt version} component is indeed to specify a +tag/commit in the repository. An example is: +\begin{verbatim} + benruijl/forcer@v1.0.0 +\end{verbatim} + +If the package is available in the system, or both the downloading and +installation succeed, then the preprocessor prepends the package path to the +\FORM{} path. Next, the package file is opened and read as in the \#include +instruction. By default, the header file that has the same name as the package, +{\tt name.h}, will be included. Alternatively, one can specify which file +should be included (with an optional foldname) by writing it after a colon +({\tt :}). The optional $+$ or $-$ sign is, as in \#include, to control the +listing of the contents of the file. Examples: +\begin{verbatim} + #usepackage benruijl/forcer@v1.0.0 +\end{verbatim} + +\begin{verbatim} + #usepackage benruijl/forcer@v1.0.0 : forcer/forcer-aux.h # decl +\end{verbatim} + +%--#] usepackage : %--#[ write : \section{\#write} diff --git a/doc/manual/startup.tex b/doc/manual/startup.tex index 4aeb9c61..a2c80408 100644 --- a/doc/manual/startup.tex +++ b/doc/manual/startup.tex @@ -31,6 +31,9 @@ \chapter{Running FORM} \item[-F] Output only to log file. Further like -L or -ll. \item[-h] Wait for some key to be touched before finishing the run. Basically only for some old window based systems. +\item[-install] Install the package\index{package!manager} specified by the + next argument. The package is specified in the format described in + \#usepackage (\ref{preusepackage}). \item[-I] Next argument/option is the path of a directory for include, procedure and subroutine files. \item[-l] Make a regular log file.