diff --git a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java index fbb88d3653f..0ae68f807d2 100644 --- a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java +++ b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java @@ -450,6 +450,7 @@ public static Throw io(String msg) { } private static String mapIOException(IOException ex) { + var msg = ex.getMessage(); if (ex instanceof FileSystemException) { // nio exceptions lack proper messages, they are encoded in the class name @@ -471,7 +472,7 @@ private static String mapIOException(IOException ex) { return "No such file: " + msg; } // otherwise fallback to the message - return msg; + return msg + (ex.getCause() != null ? (", due to: " + ex.getCause().getMessage()) : ""); } public static Throw io(IOException ex) { diff --git a/src/org/rascalmpl/library/Content.rsc b/src/org/rascalmpl/library/Content.rsc index f7615a63368..8ca87cef951 100644 --- a/src/org/rascalmpl/library/Content.rsc +++ b/src/org/rascalmpl/library/Content.rsc @@ -3,6 +3,7 @@ module Content import lang::json::IO; +import IO; @synopsis{Content wraps the HTTP Request/Response API to support interactive visualization types on the terminal.} @@ -39,6 +40,36 @@ data Content | content(Response response, str title="*static content*", ViewColumn viewColumn = normalViewColumn(1)) ; +@synopsis{A static map with default MIME interpretations for particular file extensions.} +public map[str extension, str mimeType] mimeTypes = ( + "json" :"application/json", + "css" : "text/css", + "htm" : "text/html", + "html" : "text/html", + "xml" : "text/xml", + "java" : "text/x-java-source, text/java", + "txt" : "text/plain", + "asc" : "text/plain", + "ico" : "image/x-icon", + "gif" : "image/gif", + "jpg" : "image/jpeg", + "jpeg" : "image/jpeg", + "png" : "image/png", + "mp3" : "audio/mpeg", + "m3u" : "audio/mpeg-url", + "mp4" : "video/mp4", + "ogv" : "video/ogg", + "flv" : "video/x-flv", + "mov" : "video/quicktime", + "swf" : "application/x-shockwave-flash", + "js" : "application/javascript", + "pdf" : "application/pdf", + "doc" : "application/msword", + "ogg" : "application/x-ogg", + "zip" : "application/octet-stream", + "exe" : "application/octet-stream", + "class" : "application/octet-stream" + ); @synopsis{Directly serve a static html page} Content html(str html) = content(response(html)); @@ -51,9 +82,6 @@ Content file(loc src) = content(response(src)); @synopsis{Directly serve the contents of a string as plain text} Content plainText(str text) = content(plain(text)); -alias Body = value (type[value] expected); - - @synopsis{Request values represent what a browser is asking for, most importantly the URL path.} @description{ A request value also contains the full HTTP headers, the URL parameters as a `map[str,str]` @@ -71,22 +99,77 @@ data Request (map[str, str] headers = (), map[str, str] parameters = (), map[str | head(str path) ; - @synopsis{A response encodes what is send back from the server to the browser client.} @description{ The three kinds of responses, encode either content that is already a `str`, some file which is streamed directly from its source location or a jsonResponse which involves a handy, automatic, encoding of Rascal values into json values. } -data Response - = response(Status status, str mimeType, map[str,str] header, str content) - | fileResponse(loc file, str mimeType, map[str,str] header) - | jsonResponse(Status status, map[str,str] header, value val, str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", JSONFormatter[value] formatter = str (value _) { fail; }, - bool explicitConstructorNames=false, bool explicitDataTypes=false, bool dateTimeAsInt=true, bool rationalsAsString=false) +data Response = response(Status, str mimeType, map[str, str] header, Body body); + +@synopsis{Bodies can be sent or received, depending on the context (client or)} +@description{ +* put and post requests, when received by a server, receive bodies. +* put and post requests, when fetched by a client, sent bodies. +* a response y a server sends a body. +* a response that was fetched by a client receives a body. + +The ((BodyKind)) encodes what we expect from the sender when it +puts the value onto the socket, and what we expect from the receiver +when it reads the contents off the socket. This is where builtin +conversions (formatters, parsers and validators) are activated on +the bridge between Rascal and the HTTP protocol. +} +data Body + = send(BodyKind kind, value source) + | receive(&T (BodyKind kind, type[&T] expect) receiver) + ; + +@synopsis{The type's of ((Body)) that we are sending or expecting to receive} +@description{ +This interface bridges Rascal data to the HTTP protocol. Typically large input +such as (composite) strings and JSON code is _streamed_ onto the HTTP socket. +} +data BodyKind + = text() + | json(JSONOptions options=jsonOptions()) + | file(loc storage=|unknown:///|) + ; + +data JSONOptions + = jsonOptions( + str dateTimeFormat = "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", + JSONFormatter[value] formatter = str (value _) { fail; }, + bool explicitConstructorNames=false, + bool explicitDataTypes=false, + bool dateTimeAsInt=true, + bool rationalsAsString=false + ); + +@synopsis{Convenience function for construction a JSON response value} +Response jsonResponse(Status status, map[str,str] header, value val, JSONOptions options = jsonOptions()) + = response(status, "application/json", header, send(json(options=options), val)); + +@synopsis{Convenience function for construction a text response value} +Response response(Status status, str mimeType, map[str,str] header, str content) + = response(status, mimeType, header, send(text(), content)); + +@synopsis{Convenience function for file response value} +Response fileResponse(loc source, str mimeType, map[str,str] header) + = exists(source) + ? response(ok(), mimeType, header, send(file(), source)) + : response(notFound(), "text/plain", (), send(text(), " not found.")) + ; + +@synopsis{Convenience function for construction a file response value with automatic mimetype} +Response fileResponse(loc source, map[str,str] header) + = exists(source) + ? response(ok(), mimeTypes[source.extension]?"text/plain", header, send(file(), source)) + : response(notFound(), "text/plain", (), send(text(), " not found.")) ; - + @synopsis{Utility to quickly render a string as HTML content} -Response response(str content, map[str,str] header = ()) = response(ok(), "text/html", header, content); +Response response(str content, map[str,str] header = ()) = response(ok(), "text/html", header, send(text(), content)); @synopsis{Utility to quickly report an HTTP error with a user-defined message} Response response(Status status, str explanation, map[str,str] header = ()) = response(status, "text/plain", header, explanation); @@ -108,56 +191,44 @@ default Response response(value val, map[str,str] header = ()) = jsonResponse(o @benefits{ Fast way of producing JSON strings for embedded DSLs on the Rascal side. } -Response response(value val, JSONFormatter[value] formatter, map[str,str] header = ()) = jsonResponse(ok(), header, val, formatter=formatter); +Response response(value val, JSONFormatter[value] formatter, map[str,str] header = ()) = jsonResponse(ok(), header, val, options=jsonOptions(formatter=formatter)); @synopsis{Encoding of HTTP status} -data Status - = ok() - | created() - | accepted() - | noContent() - | partialContent() - | redirect() - | notModified() - | badRequest() - | unauthorized() - | forbidden() - | notFound() - | rangeNotSatisfiable() - | internalError() - ; - +data Status + = ok() + | notFound() + | accepted() + | badRequest() + | conflict() + | created() + | expectationFailed() + | forbidden() + | found() + | gone() + | internalError() + | lengthRequired() + | methodNotAllowed() + | multiStatus() + | notAcceptible() + | notImplemented() + | notModified() + | noContent() + | partialContent() + | payloadTooLarge() + | preconditionFailed() + | rangeNotSatisfiable() + | redirect() + | redirectSeeOther() + | requestTimeout() + | serviceUnavailable() + | switchProtocol() + | temporaryRedirect() + | tooManyRequests() + | unauthorized() + | unsupportedHTTPVersion() + | unsupportedMediaType() + ; -@synopsis{A static map with default MIME interpretations for particular file extensions.} -public map[str extension, str mimeType] mimeTypes = ( - "json" :"application/json", - "css" : "text/css", - "htm" : "text/html", - "html" : "text/html", - "xml" : "text/xml", - "java" : "text/x-java-source, text/java", - "txt" : "text/plain", - "asc" : "text/plain", - "ico" : "image/x-icon", - "gif" : "image/gif", - "jpg" : "image/jpeg", - "jpeg" : "image/jpeg", - "png" : "image/png", - "mp3" : "audio/mpeg", - "m3u" : "audio/mpeg-url", - "mp4" : "video/mp4", - "ogv" : "video/ogg", - "flv" : "video/x-flv", - "mov" : "video/quicktime", - "swf" : "application/x-shockwave-flash", - "js" : "application/javascript", - "pdf" : "application/pdf", - "doc" : "application/msword", - "ogg" : "application/x-ogg", - "zip" : "application/octet-stream", - "exe" : "application/octet-stream", - "class" : "application/octet-stream" - ); @synopsis{Hint the IDE where to open the next web view or editor} @description{ diff --git a/src/org/rascalmpl/library/util/Webclient.java b/src/org/rascalmpl/library/util/Webclient.java new file mode 100644 index 00000000000..8dbc107b335 --- /dev/null +++ b/src/org/rascalmpl/library/util/Webclient.java @@ -0,0 +1,613 @@ +package org.rascalmpl.library.util; + +import java.io.BufferedOutputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublisher; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.Flow.Subscriber; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.io.input.ReaderInputStream; +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.library.Prelude; +import org.rascalmpl.library.lang.json.internal.JsonValueReader; +import org.rascalmpl.library.lang.json.internal.JsonValueWriter; +import org.rascalmpl.types.RascalTypeFactory; +import org.rascalmpl.uri.URIResolverRegistry; +import org.rascalmpl.uri.URIUtil; +import org.rascalmpl.values.IRascalValueFactory; +import org.rascalmpl.values.functions.IFunction; + +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +import fi.iki.elonen.NanoHTTPD.Response.Status; +import io.usethesource.vallang.IBool; +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IMap; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IString; +import io.usethesource.vallang.ITuple; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.type.Type; +import io.usethesource.vallang.type.TypeFactory; +import io.usethesource.vallang.type.TypeStore; + +public class Webclient { + private final IRascalValueFactory vf; + private final IRascalMonitor monitor; + private final TypeStore store; + private final TypeFactory tf; + // private final ExecutorService executor; + private final HttpClient.Builder client; + + public Webclient(IRascalValueFactory vf, IRascalMonitor monitor, TypeStore store, TypeFactory tf) { + this.vf = vf; + this.monitor = monitor; + this.store = store; + this.tf = tf; + // this.executor = Executors.newCachedThreadPool(); + this.client = HttpClient.newBuilder(); + } + + private String[] makeHeaders(IMap headers) { + return headers + .stream() + .map(ITuple.class::cast) + .flatMap((ITuple t) -> Stream.of( + ((IString) t.get(0)).getValue(), + ((IString) t.get(1)).getValue() + )) + .toArray(String[]::new) + ; + } + + private HttpRequest makeGetRequest(IConstructor input, URI uri, String[] headers) { + return HttpRequest.newBuilder(uri) + .headers(headers) + .GET() + .build(); + } + + private HttpRequest makePutRequest(IConstructor input, URI uri, String[] headers, String charset) { + return HttpRequest.newBuilder(uri) + .headers(headers) + .PUT(publishBody(input, charset)) + .build(); + } + + private HttpRequest makeDeleteRequest(IConstructor input, URI uri, String[] headers) { + return HttpRequest.newBuilder(uri) + .headers(headers) + .DELETE() + .build(); + } + + private URI requireHost(IConstructor input) { + ISourceLocation host = (ISourceLocation) input.asWithKeywordParameters().getParameter("host"); + IString path = (IString) input.get("path"); + + if (host == null) { + throw RuntimeExceptionFactory.illegalArgument(input, "missing `host` field"); + } + + if (!host.getPath().equals("/") && !host.getPath().equals("")) { + throw RuntimeExceptionFactory.illegalArgument(host, "path after hostname should be given with the \'path\' parameter of a request"); + } + + // TODO: query, fragment + return URIUtil.getChildLocation(host, path.getValue()).getURI(); + } + + private HttpRequest makeHeadRequest(IConstructor input, URI uri, String[] headers) { + return HttpRequest.newBuilder(uri) + .headers(headers) + .method("HEAD", BodyPublishers.noBody()) + .build(); + } + + private BodyPublisher publishJsonBody(IConstructor input, String charset) { + return new BodyPublisher() { + @Override + public void subscribe(Subscriber subscriber) { + // executor.submit(() -> { + IConstructor options = input.asWithKeywordParameters().getParameter("options"); + Map kws = options != null + ? options.asWithKeywordParameters().getParameters() + : Collections.emptyMap(); + IString dtf = (IString) kws.get("dateTimeFormat"); + IBool dai = (IBool) kws.get("dateTimeAsInt"); + IBool ras = (IBool) kws.get("rationalsAsString"); + IFunction formatters = (IFunction) kws.get("formatter"); + IBool ecn = (IBool) kws.get("explicitConstructorNames"); + IBool edt = (IBool) kws.get("explicitDataTypes"); + + JsonValueWriter writer = new JsonValueWriter() + .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") + .setFormatters(formatters) + .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true) + .setRationalsAsString(ras != null ? ((IBool) ras).getValue() : false) + .setExplicitConstructorNames(ecn != null ? ((IBool) ecn).getValue() : false) + .setExplicitDataTypes(edt != null ? ((IBool) edt).getValue() : false) + ; + + try (JsonWriter jsonWriter = new JsonWriter(new OutputStreamWriter(new OutputStreamPublisher(subscriber), charset))) { + writer.write(jsonWriter, input); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + // }, true); + } + + @Override + public long contentLength() { + return -1; + } + }; + } + + private BodyPublisher publishFileBody(IConstructor kind, IConstructor input) { + final var loc = (ISourceLocation) input.get("source"); + + // have to hope that the encoding matches the actual contents of the stream + + return BodyPublishers.ofInputStream(() -> { + try { + return URIResolverRegistry.getInstance().getInputStream(loc); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + }); + } + + private BodyPublisher publishTextBody(IConstructor input, String charset) { + final var text = (IString) input.get("source"); + return BodyPublishers.ofInputStream(() -> new ReaderInputStream(text.asReader(), charset)); + } + + /** + * This is for PUT and POST requests that need to send data along with the request + */ + private BodyPublisher publishBody(IConstructor input, String charset) { + var postBody = (IConstructor) input.get("content"); + + if (!postBody.getName().equals("send")) { + throw RuntimeExceptionFactory.illegalArgument(postBody, "Client-side POST should send a Body, not receive one"); + } + + final var kind = (IConstructor) postBody.get("kind"); + + switch (kind.getName()) { + case "json": + return publishJsonBody(postBody, charset); + case "file": + return publishFileBody(kind, postBody); + case "text": + return publishTextBody(postBody, charset); + default: + return null; + } + } + + /** + * On demand streamer for the HttpClient API (for sending bodies for PUT and POST) + */ + private static class OutputStreamPublisher extends BufferedOutputStream { + + public OutputStreamPublisher(Subscriber subscriber) { + super(new PublishingStream(subscriber)); + } + /** + * The buffed outputstream will take care to collect the bytes untill there's a decent chunk to forward to the consumers + */ + private static class PublishingStream extends OutputStream { + private final Subscriber subscriber; + + public PublishingStream(Subscriber subscriber) { + this.subscriber = subscriber; + } + + @Override + public void write(int b) throws IOException { + subscriber.onNext(ByteBuffer.wrap(new byte[] { (byte)(b & 0xFF) })); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + subscriber.onNext(ByteBuffer.wrap(b, off, len).asReadOnlyBuffer()); + } + + @Override + public void close() throws IOException { + subscriber.onComplete(); + } + } + } + + private HttpRequest makePostRequest(IConstructor input, URI uri, String[] headers, String charset) { + return HttpRequest.newBuilder() + .uri(uri) + .headers(headers) + .POST(publishBody(input, charset)) + .build(); + } + + private String defaultContentType(IConstructor input) { + if (input.getName().equals("post") || input.getName().equals("put")) { + var body = (IConstructor) input.get("content"); + var kind = (IConstructor) body.get("kind"); + + switch (kind.getName()) { + case "json": + return "application/json"; + case "file": + return "application/octet-stream"; + case "text": + default: + return "text/plain"; + } + } + + // only post and put have content to send + return ""; + } + + private HttpRequest makeRequest(IConstructor input) { + var host = requireHost(input); + var headers = (IMap) input.asWithKeywordParameters().getParameter("header"); + + if (headers == null) { + headers = vf.map(); + } + + var charset = ((IString) input.asWithKeywordParameters().getParameter("charset")); + + if (charset == null) { + charset = vf.string(StandardCharsets.UTF_8.name()); + } + + var contentType = ((IString) input.asWithKeywordParameters().getParameter("content-type")); + + if (contentType == null) { + contentType = vf.string(defaultContentType(input)); + } + + if (headers.get(vf.string("Content-Type")) != null) { + throw RuntimeExceptionFactory.illegalArgument(input, "For POST and PUT, use the keyword fields 'content-type' and 'charset' instead of the 'Content-Type' header field."); + } + + if (contentType.length() != 0) { + headers = headers.put(vf.string("Content-Type"), vf.string(contentType + ";charset=" + charset)); + } + + // need at least one header to avoid IllegalArgumentExceptions by the HTTP builder + headers = headers.put(vf.string("User-Agent"), vf.string("rascal-stdlib")); + + var httpHeaders = makeHeaders(headers); + + switch (input.getName()) { + case "get": + return makeGetRequest(input, host, httpHeaders); + case "post": + return makePostRequest(input, host, httpHeaders, charset.getValue()); + case "put": + return makePutRequest(input, host, httpHeaders, charset.getValue()); + case "delete": + return makeDeleteRequest(input, host, httpHeaders); + case "head": + return makeHeadRequest(input, host, httpHeaders); + + default: + throw RuntimeExceptionFactory.illegalArgument(input); + } + } + + /** + * This is the main API method for the Rascal side + */ + public IConstructor fetch(IConstructor input) { + try { + var request = makeRequest(input); + var response = client + .followRedirects(HttpClient.Redirect.NORMAL) + .build() + .send(request, HttpResponse.BodyHandlers.ofInputStream()); + var body = (IConstructor) input.asWithKeywordParameters().getParameter("body"); + + return translateResponse(request.uri().toString(), body, response); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } + } + + /* + * This creates a Body::receive() constructor that wraps a lambda to call and receive the data + * from the HTTP response as a Rascal value. The intermediate step is required such that the + * caller of `fetch` can express their expectations on the data and call appropriate parsing, + * validation and binding mechanisms. The options are encoded as `BodyKind`. + * + * Also this generated function asks for a reified type parameter for (dynamic) type safety + * and validation of abstract grammars during reception of the body. + */ + private IFunction createBodyReceiver(InputStream input, String url, String contentType, String charset) { + var BodyKind = store.lookupAbstractDataType("BodyKind"); + var rt = RascalTypeFactory.getInstance().reifiedType(tf.parameterType("T")); + var ft = tf.functionType(tf.parameterType("T"), tf.tupleType(BodyKind, rt), tf.tupleEmpty()); + + return vf.function(ft, (args, kwargs) -> { + IConstructor kind = (IConstructor) args[0]; + IConstructor reified = (IConstructor) args[1]; + Type expect = reified.getType().getTypeParameters().getFieldType(0); + + switch (kind.getName()) { + case "json": + return receiveJsonBody(input, url, kind, expect, contentType, charset); + case "file": + return receiveFileBody(input, kind, expect); + case "text": + default: + return receiveTextBody(input, kind, expect, contentType, charset); + } + }); + } + + private IValue receiveTextBody(InputStream input, IConstructor kind, Type expect, String contentType, String charset) { + if (!expect.isSubtypeOf(tf.stringType())) { + throw RuntimeExceptionFactory.illegalArgument(kind, "a text response expects a `str` type"); + } + + if (!contentType.contains("text/")) { + throw RuntimeExceptionFactory.illegalArgument(kind, "a text response was expected but the mimetype is " + contentType); + } + + try { + return vf.string(Prelude.consumeInputStream(new InputStreamReader(input, charset))); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + } + + private IValue receiveFileBody(InputStream input, IConstructor kind, Type expect) { + ISourceLocation loc = (ISourceLocation) kind.get("storage"); + + // note that mimetype and charset are kept as-is during the download directly to disk + try (OutputStream out = URIResolverRegistry.getInstance().getOutputStream(loc, false)) { + input.transferTo(out); + return loc; + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + } + + private IValue receiveJsonBody(InputStream input, String url, IConstructor kind, Type expect, String contentType, String charset) { + if (!contentType.equals("application/json")) { + throw RuntimeExceptionFactory.illegalArgument(kind, "expected content-type 'application/json', got: " + contentType); + } + + try { + JsonReader jsonReader = new JsonReader(new InputStreamReader(input, charset)); + JsonValueReader parser = new JsonValueReader(vf, store, monitor, URIUtil.assumeCorrectLocation(url)); + return parser.read(jsonReader, expect); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + } + + private IConstructor translateResponse(String url, IConstructor expect, HttpResponse response) throws IOException { + var headers = response + .headers() + .map() + .entrySet() + .stream() + .map(e -> vf.tuple( + vf.string(e.getKey()), + vf.string(e.getValue().stream().collect(Collectors.joining(",")) + ))) + .collect(vf.mapWriter()); + + long totalBytes = response.headers() + .firstValueAsLong("Content-Length") + .orElse(-1); + + var input = totalBytes > 0 + ? new MonitoredInputStream(response.body(), monitor, "Fetching " + url, totalBytes) + : response.body(); + + String mimeType = getMimeType(response.headers()); + String charset = getCharset(response.headers()); + var status = toStatusConstructor(response.statusCode()); + + Type respCons = store.lookupConstructors("response").iterator().next(); + IFunction bodyReceiver = createBodyReceiver(input, url, mimeType, charset); + Type bodyConstructor = store.lookupConstructors("receive").iterator().next(); + IConstructor body = vf.constructor(bodyConstructor, bodyReceiver); + + return vf.constructor(respCons, status, vf.string(mimeType), headers, body); + } + + private String getCharset(HttpHeaders headers) { + String contentType = headers.firstValue("Content-Type").orElse("text/plain"); + String result = StandardCharsets.UTF_8.name(); + + String[] parts = contentType.split(";"); + if (parts.length > 1) { + String[] assign = parts[1].split("="); + + if (assign[0].equals("charset")) { + result = assign[1].trim(); + } + } + + return result; + } + + private String getMimeType(HttpHeaders headers) { + String contenType = headers.firstValue("Content-Type").orElse("text/plain"); + String[] parts = contenType.split(";"); + return parts[0].trim(); + } + + private IConstructor toStatusConstructor(int stCode) { + var statusType = store.lookupAbstractDataType("Status"); + + var status = Status.lookup(stCode); + switch (status) { + case OK: + return vf.constructor(store.lookupConstructor(statusType, "ok", tf.tupleEmpty())); + case NOT_FOUND: + return vf.constructor(store.lookupConstructor(statusType, "notFound", tf.tupleEmpty())); + case ACCEPTED: + return vf.constructor(store.lookupConstructor(statusType, "accepted", tf.tupleEmpty())); + case BAD_REQUEST: + return vf.constructor(store.lookupConstructor(statusType, "badRequest", tf.tupleEmpty())); + case CONFLICT: + return vf.constructor(store.lookupConstructor(statusType, "conflict", tf.tupleEmpty())); + case CREATED: + return vf.constructor(store.lookupConstructor(statusType, "create", tf.tupleEmpty())); + case EXPECTATION_FAILED: + return vf.constructor(store.lookupConstructor(statusType, "expectationFailed", tf.tupleEmpty())); + case FORBIDDEN: + return vf.constructor(store.lookupConstructor(statusType, "forbidden", tf.tupleEmpty())); + case FOUND: + return vf.constructor(store.lookupConstructor(statusType, "found", tf.tupleEmpty())); + case GONE: + return vf.constructor(store.lookupConstructor(statusType, "gone", tf.tupleEmpty())); + case INTERNAL_ERROR: + return vf.constructor(store.lookupConstructor(statusType, "internalError", tf.tupleEmpty())); + case LENGTH_REQUIRED: + return vf.constructor(store.lookupConstructor(statusType, "lengthRequired", tf.tupleEmpty())); + case METHOD_NOT_ALLOWED: + return vf.constructor(store.lookupConstructor(statusType, "methodNotAllowed", tf.tupleEmpty())); + case MULTI_STATUS: + return vf.constructor(store.lookupConstructor(statusType, "multiStatus", tf.tupleEmpty())); + case NOT_ACCEPTABLE: + return vf.constructor(store.lookupConstructor(statusType, "notAcceptible", tf.tupleEmpty())); + case NOT_IMPLEMENTED: + return vf.constructor(store.lookupConstructor(statusType, "notImplemented", tf.tupleEmpty())); + case NOT_MODIFIED: + return vf.constructor(store.lookupConstructor(statusType, "notModified", tf.tupleEmpty())); + case NO_CONTENT: + return vf.constructor(store.lookupConstructor(statusType, "noContent", tf.tupleEmpty())); + case PARTIAL_CONTENT: + return vf.constructor(store.lookupConstructor(statusType, "partialContent", tf.tupleEmpty())); + case PAYLOAD_TOO_LARGE: + return vf.constructor(store.lookupConstructor(statusType, "payloadTooLarge", tf.tupleEmpty())); + case PRECONDITION_FAILED: + return vf.constructor(store.lookupConstructor(statusType, "preconditionFailed", tf.tupleEmpty())); + case RANGE_NOT_SATISFIABLE: + return vf.constructor(store.lookupConstructor(statusType, "rangeNotSatisfieable", tf.tupleEmpty())); + case REDIRECT: + return vf.constructor(store.lookupConstructor(statusType, "redirect", tf.tupleEmpty())); + case REDIRECT_SEE_OTHER: + return vf.constructor(store.lookupConstructor(statusType, "redirectSeeOther", tf.tupleEmpty())); + case REQUEST_TIMEOUT: + return vf.constructor(store.lookupConstructor(statusType, "requestTimeout", tf.tupleEmpty())); + case SERVICE_UNAVAILABLE: + return vf.constructor(store.lookupConstructor(statusType, "serviceUnavailable", tf.tupleEmpty())); + case SWITCH_PROTOCOL: + return vf.constructor(store.lookupConstructor(statusType, "switchProtocol", tf.tupleEmpty())); + case TEMPORARY_REDIRECT: + return vf.constructor(store.lookupConstructor(statusType, "temporaryRedirect", tf.tupleEmpty())); + case TOO_MANY_REQUESTS: + return vf.constructor(store.lookupConstructor(statusType, "tooManyRequests", tf.tupleEmpty())); + case UNAUTHORIZED: + return vf.constructor(store.lookupConstructor(statusType, "unauthorized", tf.tupleEmpty())); + case UNSUPPORTED_HTTP_VERSION: + return vf.constructor(store.lookupConstructor(statusType, "unsupportedHTTPVersion", tf.tupleEmpty())); + case UNSUPPORTED_MEDIA_TYPE: + return vf.constructor(store.lookupConstructor(statusType, "unsupportedMediaType", tf.tupleEmpty())); + default: + // if we don't understand the error code; let's call it an internal error + return vf.constructor(store.lookupConstructor(statusType, "internalError", tf.tupleEmpty())); + } + } + + private class MonitoredInputStream extends FilterInputStream { + private final IRascalMonitor monitor; + private final String jobName; + + private final long totalBytes; + private long bytesRead = 0; + private boolean started = false; + private boolean done = false; + + public MonitoredInputStream(InputStream in, IRascalMonitor monitor, String jobName, long totalBytes) { + super(in); + this.totalBytes = totalBytes; + this.monitor = monitor; + this.jobName = jobName; + } + + private void ensureStarted() { + if (!started) { + started = true; + monitor.jobStart(jobName, Integer.MAX_VALUE); + } + } + + private void updateProgress(int bytesRead) throws InterruptedIOException { + if (monitor.jobIsCanceled(jobName)) { + throw new InterruptedIOException(jobName); + } + + ensureStarted(); + long numberOfTheseSteps = (int) (totalBytes / bytesRead); + int stepSize = (int) (Integer.MAX_VALUE / numberOfTheseSteps); + monitor.jobStep(jobName, "", java.lang.Math.max(stepSize, 1)); + checkDone(); + } + + private void checkDone() { + if (!done && bytesRead >= totalBytes) { + done = true; + monitor.jobEnd(jobName, true); + } + } + + @Override + public int read() throws IOException { + int b = super.read(); + if (b != -1) { + bytesRead += 1; + updateProgress(1); + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int n = super.read(b, off, len); + if (n > 0) { + bytesRead += n; + updateProgress(n); + } + return n; + } + } + } diff --git a/src/org/rascalmpl/library/util/Webclient.rsc b/src/org/rascalmpl/library/util/Webclient.rsc new file mode 100644 index 00000000000..192455c8694 --- /dev/null +++ b/src/org/rascalmpl/library/util/Webclient.rsc @@ -0,0 +1,34 @@ +module util::Webclient + +extend Content; +import Exception; + +private str DEFAULT_CHARSET = "UTF-8"; + +@synopsis{From the client side, a host name is required and also an expected content-type} +@description{ +* `host` is the essential information, the URL to fetch input from. The path will be retrieved from the `path` parameter of `get`, `put`, `post` and `head` +* `content-type` is relevant for POST and PUT options where content is uploaded to the server. The mimetype will be set to this value. +* `charset` is also relevant for POST and PUT options, the outgoing stream will be encoded accordingly and the headers will contained the right meta-data. +} +data Request(loc host = hostIsRequired(), str \content-type = "text/plain", str charset=DEFAULT_CHARSET); + +@javaClass{org.rascalmpl.library.util.Webclient} +java Response fetch(Request request); + +@synopsis{Short-hand for construction of JSON post bodies} +Request jsonPost(str path, value content, loc host=hostIsRequired(), JSONOptions options=jsonOptions()) + = post(path, send(json(options=options), content), host=host, \content-type="application/json"); + +@synopsis{Short-hand for construction of JSON post bodies} +Request jsonPut(str path, value content, loc host=hostIsRequired(), JSONOptions options=jsonOptions()) + = put(path, send(json(options=options), content), host=host, \content-type="application/json"); + +@synopsis{Short-hand for construction of JSON post bodies} +Request filePut(str path, loc content, loc host=hostIsRequired()) + = put(path, send(file(), content), host=host, \content-type="application/json"); + + +private loc hostIsRequired() throws IllegalArgument { + throw IllegalArgument("missing host parameter"); +} diff --git a/src/org/rascalmpl/library/util/Webserver.java b/src/org/rascalmpl/library/util/Webserver.java index 5dd103a0b5e..d8a885cf45f 100644 --- a/src/org/rascalmpl/library/util/Webserver.java +++ b/src/org/rascalmpl/library/util/Webserver.java @@ -379,11 +379,7 @@ private IMap makeMap(Map headers) { throw RuntimeExceptionFactory.io(e); } } - - - - - + public void shutdown(ISourceLocation server) { NanoHTTPD nano = servers.get(server); if (nano != null) { diff --git a/src/org/rascalmpl/repl/rascal/RascalReplServices.java b/src/org/rascalmpl/repl/rascal/RascalReplServices.java index d732c71cfba..b7421a612c6 100644 --- a/src/org/rascalmpl/repl/rascal/RascalReplServices.java +++ b/src/org/rascalmpl/repl/rascal/RascalReplServices.java @@ -43,7 +43,6 @@ import org.rascalmpl.parser.gtd.exception.ParseError; import org.rascalmpl.repl.IREPLService; import org.rascalmpl.repl.StopREPLException; -import org.rascalmpl.repl.TerminalProgressBarMonitor; import org.rascalmpl.repl.completers.CompletionMatcherWithEscapes; import org.rascalmpl.repl.completers.RascalCommandCompletion; import org.rascalmpl.repl.completers.RascalIdentifierCompletion; diff --git a/src/org/rascalmpl/semantics/dynamic/Statement.java b/src/org/rascalmpl/semantics/dynamic/Statement.java index 315acc84e6f..b078fae719f 100644 --- a/src/org/rascalmpl/semantics/dynamic/Statement.java +++ b/src/org/rascalmpl/semantics/dynamic/Statement.java @@ -48,7 +48,6 @@ import org.rascalmpl.interpreter.staticErrors.UninitializedVariable; import org.rascalmpl.interpreter.utils.Cases; import org.rascalmpl.interpreter.utils.Cases.CaseBlock; -import org.rascalmpl.uri.URIUtil; import org.rascalmpl.interpreter.utils.Names; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IInteger;