|
28 | 28 | import com.github.underscore.PredicateIndexed; |
29 | 29 | import com.github.underscore.Tuple; |
30 | 30 | import com.github.underscore.Underscore; |
| 31 | +import java.io.File; |
| 32 | +import java.io.FileInputStream; |
| 33 | +import java.io.FileOutputStream; |
| 34 | +import java.io.IOException; |
| 35 | +import java.net.URL; |
| 36 | +import java.nio.channels.Channels; |
| 37 | +import java.nio.channels.ReadableByteChannel; |
31 | 38 | import java.nio.charset.StandardCharsets; |
| 39 | +import java.nio.file.Files; |
| 40 | +import java.nio.file.Paths; |
32 | 41 | import java.util.ArrayList; |
33 | 42 | import java.util.Arrays; |
34 | 43 | import java.util.Collection; |
|
48 | 57 | import java.util.function.Consumer; |
49 | 58 | import java.util.function.Function; |
50 | 59 | import java.util.function.Predicate; |
| 60 | +import java.util.zip.GZIPInputStream; |
51 | 61 |
|
52 | 62 | public class U<T> extends Underscore<T> { |
53 | 63 | private static final int DEFAULT_TRUNC_LENGTH = 30; |
@@ -1948,6 +1958,22 @@ public Map<String, Object> xmlMap() { |
1948 | 1958 | } |
1949 | 1959 | } |
1950 | 1960 |
|
| 1961 | + public static long downloadUrl(final String url, final String fileName) throws IOException { |
| 1962 | + final URL website = new URL(url); |
| 1963 | + try (ReadableByteChannel rbc = Channels.newChannel(website.openStream()); |
| 1964 | + final FileOutputStream fos = new FileOutputStream(fileName)) { |
| 1965 | + return fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); |
| 1966 | + } |
| 1967 | + } |
| 1968 | + |
| 1969 | + public static void decompressGzip(final String sourceFileName, final String targetFileName) |
| 1970 | + throws IOException { |
| 1971 | + try (GZIPInputStream gis = |
| 1972 | + new GZIPInputStream(new FileInputStream(new File(sourceFileName)))) { |
| 1973 | + Files.copy(gis, Paths.get(targetFileName)); |
| 1974 | + } |
| 1975 | + } |
| 1976 | + |
1951 | 1977 | public static FetchResponse fetch(final String url) { |
1952 | 1978 | return fetch(url, null, null, DEFAULT_HEADER_FIELDS, null, null); |
1953 | 1979 | } |
@@ -2156,8 +2182,14 @@ && nonNull(timeBetweenRetry) |
2156 | 2182 | UnsupportedOperationException saveException; |
2157 | 2183 | do { |
2158 | 2184 | try { |
2159 | | - final FetchResponse fetchResponse = U.fetch( |
2160 | | - url, method, body, headerFields, connectTimeout, readTimeout); |
| 2185 | + final FetchResponse fetchResponse = |
| 2186 | + U.fetch( |
| 2187 | + url, |
| 2188 | + method, |
| 2189 | + body, |
| 2190 | + headerFields, |
| 2191 | + connectTimeout, |
| 2192 | + readTimeout); |
2161 | 2193 | if (fetchResponse.getStatus() == 429) { |
2162 | 2194 | saveException = new UnsupportedOperationException("Too Many Requests"); |
2163 | 2195 | } else { |
|
0 commit comments