diff --git a/.gitignore b/.gitignore index 61f66bd..d331f0e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ # vim undo files *~ + +# require 'remote GET' +.require-remote-deps/ diff --git a/src/log.q b/src/log.q index 726bce5..eba3bf5 100644 --- a/src/log.q +++ b/src/log.q @@ -73,7 +73,7 @@ .log.pattern:()!(); / When call line tracing is enabled, this list of strings can be used to remove common prefixes from the file paths. By default, if this is -/ empty when the library is initialised, it will be defaulted to '.require.location.root' +/ empty when the library is initialised, it will be defaulted to '.require.location.paths' .log.sourcePathExcludePrefixes:(); @@ -91,7 +91,7 @@ ]; if[0 = count .log.sourcePathExcludePrefixes; - .log.sourcePathExcludePrefixes,:enlist 1_ string .require.location.root; + .log.sourcePathExcludePrefixes,:1_/: string .require.location.paths; ]; / setLogger calls setLevel diff --git a/src/require.q b/src/require.q index a3457be..3105270 100644 --- a/src/require.q +++ b/src/require.q @@ -9,15 +9,17 @@ / Table containing the state of each library loaded via require .require.loadedLibs:`lib xkey flip `lib`loaded`loadedTime`inited`initedTime`files!"SBPBP*"$\:(); -/ Root folder to search for libraries -.require.location.root:`; + +/ Search paths for 'require' libraries. Paths will be descended through (similar to 'tree') +.require.location.paths:`symbol$(); / Regexs to filter discovered files / @see .require.i.tree .require.location.ignore:("*.git";"*target"); / Complete list of discovered files from the root directory -.require.location.discovered:enlist`; +.require.location.discovered:`symbol$(); + / Required interface implementations for 'require' and related kdb-common libraries to function correctly .require.interfaces:`lib`ifFunc xkey flip `lib`ifFunc`implFunc!"SS*"$\:(); @@ -36,23 +38,20 @@ :(::); ]; - $[null root; - .require.location.root:.require.i.getCwd[]; - .require.location.root:root - ]; + .require.location.paths:.require.location.paths union (root; .require.i.getCwd[]) null root; .require.i.setDefaultInterfaces[]; (.require.markLibAsLoaded;.require.markLibAsInited)@\:`require; / If file tree has already been specified, don't overwrite - if[.require.location.discovered~enlist`; - .require.rescanRoot[]; + if[0 = count .require.location.discovered; + .require.rescan[]; ]; .require.i.initInterfaceLibrary[]; - .log.if.info "Require library initialised [ Root: ",string[.require.location.root]," ]"; + .log.if.info ("Require library initialised [ Paths: {} ]"; .require.location.paths); }; @@ -83,10 +82,29 @@ (.require.i.load;.require.i.init)@\:lib; }; -.require.rescanRoot:{ - .require.location.discovered:.require.i.tree .require.location.root; +.require.rescan:.require.rescanRoot:{ + .log.if.debug ("Rescanning all require library paths [ Paths: {} ]"; .require.location.paths); + + .require.location.discovered:raze .require.i.tree each .require.location.paths; + + .log.if.info enlist["Library files refreshed [ Paths: {} ] [ Files: {} ]"],count each .require.location`paths`discovered; + }; + +.require.addPath:{[path; rescan] + .log.if.info ("Adding new path to 'require' search path [ Path: {} ] [ Rescan: {} ]"; path; `no`yes rescan); + + .require.location.paths:.require.location.paths union path; + + if[rescan; + .require.rescan[]; + ]; + }; + +.require.removePath:{[path] + .log.if.info ("Removing path from 'require' search path [ Path: {} ]"; path); - .log.if.info "Library root location refreshed [ File Count: ",string[count .require.location.discovered]," ]"; + .require.location.paths:.require.location.paths except path; + .require.rescan[]; }; / Marks the specified library as loaded in the loaded libraries table. NOTE: This diff --git a/src/rrg.q b/src/rrg.q new file mode 100644 index 0000000..4e6b6b1 --- /dev/null +++ b/src/rrg.q @@ -0,0 +1,141 @@ +// Require Remote GET +// Copyright (c) 2021 Jaskirat Rajasansir + +.require.lib each `http`file; + +// URL FORMAT -- source://user/repo:version +// "latest" supported for 'version' + +.rrg.cfg.repoUrls:(`symbol$())!(); +.rrg.cfg.repoUrls[`gh]:"https://api.github.com/repos/{owner}/{repo}"; + +.rrg.cfg.relativeUrls:`source`query xkey flip `source`query`relativeUrl!"SS*"$\:(); +.rrg.cfg.relativeUrls[``]:enlist ""; +.rrg.cfg.relativeUrls[`gh`releases]:enlist "releases"; + +.rrg.location.root:`; + + +.rrg.init:{ + if[null .rrg.location.root; + .rrg.location.root:` sv first[.require.location.paths],`$".require-remote-deps"; + ]; + + .file.ensureDir .rrg.location.root; + + existingLibs:.rrg.list[]; + + if[0 < count existingLibs; + .require.addPath[;0b] each existingLibs`libRoot; + .require.rescan[]; + ]; + + .log.if.info ("Require 'Remote GET' library initialised [ Download Target: {} ] [ Current Libs: {} ]"; .rrg.location.root; count existingLibs); + }; + + +.rrg.get:{[request] + details:.rrg.i.parseGetRequest request; + + versions:details[`url],"/",.rrg.cfg.relativeUrls[details[`source],`releases]`relativeUrl; + + verResp:.http.get[versions; ()!()]; + + if[not `success = verResp`statusType; + .log.if.error "Failed to query API for repository information [ Request: ",request," ]"; + '"InvalidRepositoryException"; + ]; + + releases:select from verResp[`body] where not draft, not prerelease; + + if[0 = count releases; + .log.if.error "No published releases available for repository [ Request: ",request," ]"; + '"NoRepositoryVersionsException"; + ]; + + $["latest" ~ details`version; + theRelease:first releases; + / else + theRelease:first select from releases where tag_name like "*",details[`version],"*" + ]; + + if[0 = count theRelease `tarball_url; + .log.if.error "Invalid version specified (or no releases available) [ Request: ",request," ]"; + '"InvalidRepositoryVersionException"; + ]; + + tarTarget:` sv .rrg.location.root,`.staging,` sv details[`repo],`tar`gz; + + .log.if.info "Downloading repository version [ Request: ",request," ] [ Version: ",theRelease[`tag_name]," ] [ Target: ",string[tarTarget]," ]"; + + repo:.http.get[theRelease`tarball_url; ()!()]; + tarTarget 1: repo`body; + + extractTarget:` sv .rrg.location.root,details`repo; + + .log.if.info "Extracting repostiory release [ Target: ",string[extractTarget]," ]"; + + if[.type.isFolder extractTarget; + .log.if.debug "Removing existing download for repository [ Request: ",request," ]"; + .os.run[`rmFolder; 1_ string extractTarget]; + ]; + + .file.ensureDir extractTarget; + + .util.system "tar -xzf ",(1_ string tarTarget)," --strip-components=1 --directory ",1_ string extractTarget; + (` sv extractTarget,`RRG_VERSION) 1: theRelease`tag_name; + + rrgTree:.file.tree extractTarget; + kdbFiles:rrgTree where any rrgTree like/: "*",/:(".q"; ".k"; ".q_"; ".k_"); + + if[0 = count kdbFiles; + .log.if.warn ("Downloaded repository contains no kdb+ files. Not added to path [ Repository: {} ]"; request); + :(::); + ]; + + .log.if.info ("Repository downloaded and adding to 'require' path [ Repository: {} ] [ Files: {} ]"; request; count kdbFiles); + .require.addPath[extractTarget; 1b]; + }; + +.rrg.list:{ + list:flip `lib`libRoot`versionFile`version!"SSS*"$\:(); + list:list upsert ([] lib:.file.ls .rrg.location.root; libRoot:.file.listFolderPaths .rrg.location.root); + list:update versionFile:(` sv/:libRoot,\:`RRG_VERSION) from list; + list:update version:first each .ns.protectedExecute[read0;] each versionFile from list; + list:delete from list where version~\:.ns.const.pExecFailure; + + :list; + }; + +.rrg.import:{[lib; repo; relPath] + if[not .type.isFolder repo; + '"InvalidRepositoryException"; + ]; + + isGitRepo:not .ns.const.pExecFailure ~ first .ns.protectedExecute[`.util.system; "git rev-parse --is-inside-work-tree"]; + + if[not isGitRepo; + '"InvalidRepositoryException"; + ]; + + details:.rrg.i.parseGetRequest lib; + + repoDetail:.http.get[details`url; ()!()]; + + if[0 = count repoDetail[`body]`clone_url; + '"InvalidRepositoryException"; + ]; + + .log.if.info "Adding remote dependency as submodule [ Lib: ",lib," ] [ Local Git Repo: ",string[repo]," ] [ Relative Path: ",string[path]," ]"; + + -1 "git submodule add ",(repoDetail[`body]`clone_url)," ",1_ string relPath; + }; + +.rrg.i.parseGetRequest:{[req] + parsed:.http.i.getUrlDetails req; + + result:`source`owner`repo`version!(`$-3_ parsed`scheme; `$parsed`baseUrl; `$1_ first ":" vs parsed`path; last ":" vs parsed`path); + result[`url]:.util.findAndReplace[.rrg.cfg.repoUrls result`source; ("{owner}"; "{repo}"); string result`owner`repo]; + + :result; + };