From d4c79cc5bc6e499b2a29dda1e3ed84a89abb73da Mon Sep 17 00:00:00 2001 From: en-ken Date: Sun, 11 Aug 2019 20:52:54 +0900 Subject: [PATCH 01/15] feat: added fortune logic. --- kadai4/en-ken/kadai4/export_test.go | 16 +++++++ kadai4/en-ken/kadai4/fortune.go | 28 ++++++++++++ kadai4/en-ken/kadai4/fortune_test.go | 64 ++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 kadai4/en-ken/kadai4/export_test.go create mode 100644 kadai4/en-ken/kadai4/fortune.go create mode 100644 kadai4/en-ken/kadai4/fortune_test.go diff --git a/kadai4/en-ken/kadai4/export_test.go b/kadai4/en-ken/kadai4/export_test.go new file mode 100644 index 0000000..9f95ea0 --- /dev/null +++ b/kadai4/en-ken/kadai4/export_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "time" +) + +var GetFortune = getFortune +var Handler = handler + +func SetNow(n func() time.Time) { + now = n +} + +func SetFortunes(f []string) { + fortunes = f +} diff --git a/kadai4/en-ken/kadai4/fortune.go b/kadai4/en-ken/kadai4/fortune.go new file mode 100644 index 0000000..aab8c95 --- /dev/null +++ b/kadai4/en-ken/kadai4/fortune.go @@ -0,0 +1,28 @@ +package main + +import ( + "math/rand" + "time" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// For testing +var now = time.Now +var fortunes []string + +func getFortune() string { + i := rand.Int() % len(fortunes) + f := fortunes[i] + + t := now() + if t.Month() == 1 { + if t.Day() >= 1 && t.Day() <= 3 { + f = "大吉" + } + } + + return f +} diff --git a/kadai4/en-ken/kadai4/fortune_test.go b/kadai4/en-ken/kadai4/fortune_test.go new file mode 100644 index 0000000..56b80f1 --- /dev/null +++ b/kadai4/en-ken/kadai4/fortune_test.go @@ -0,0 +1,64 @@ +package main_test + +import ( + "fmt" + "testing" + "time" + + main "github.com/gopherdojo/dojo6/kadai4/en-ken/kadai4" +) + +func TestGetLuck(t *testing.T) { + + cases := []struct { + now time.Time + expected string + }{ + { + now: time.Date(2018, time.December, 31, 0, 0, 0, 0, time.Local), + expected: "凶", + }, + { + now: time.Date(2018, time.December, 31, 23, 59, 59, 999, time.Local), + expected: "凶", + }, + { + now: time.Date(2019, time.January, 1, 0, 0, 0, 0, time.Local), + expected: "大吉", + }, + { + now: time.Date(2019, time.January, 2, 0, 0, 0, 0, time.Local), + expected: "大吉", + }, + { + now: time.Date(2019, time.January, 3, 0, 0, 0, 0, time.Local), + expected: "大吉", + }, + { + now: time.Date(2018, time.January, 3, 23, 59, 59, 999, time.Local), + expected: "大吉", + }, + { + now: time.Date(2019, time.January, 4, 0, 0, 0, 0, time.Local), + expected: "凶", + }, + } + + main.SetFortunes([]string{"凶", "凶", "凶"}) + + for i, c := range cases { + c := c + t.Run( + fmt.Sprintf("case[%v]", i), + func(t *testing.T) { + main.SetNow(func() time.Time { + return c.now + }) + + if actual := main.GetFortune(); actual != c.expected { + t.Errorf("actual:%v, expected:%v\n", actual, c.expected) + } + }) + } + +} From ae34fafa23aa54c1ecd90275e24e790804602a98 Mon Sep 17 00:00:00 2001 From: en-ken Date: Sun, 11 Aug 2019 20:53:37 +0900 Subject: [PATCH 02/15] feat: added handler logic. --- kadai4/en-ken/kadai4/main.go | 33 +++++++++++++++++++++++++++++++ kadai4/en-ken/kadai4/main_test.go | 29 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 kadai4/en-ken/kadai4/main.go create mode 100644 kadai4/en-ken/kadai4/main_test.go diff --git a/kadai4/en-ken/kadai4/main.go b/kadai4/en-ken/kadai4/main.go new file mode 100644 index 0000000..8732308 --- /dev/null +++ b/kadai4/en-ken/kadai4/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "encoding/json" + "net/http" +) + +func init() { + fortunes = []string{"大吉", "吉", "中吉", "小吉", "末吉", "凶", "大凶"} +} + +type response struct { + Fortune string `json:"fortune"` +} + +func handler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "applicaiton/json; charset=utf-8") + + enc := json.NewEncoder(w) + + resp := &response{ + Fortune: getFortune(), + } + + if err := enc.Encode(resp); err != nil { + http.Error(w, "Server error", http.StatusInternalServerError) + } +} + +func main() { + http.HandleFunc("/", handler) + http.ListenAndServe(":8080", nil) +} diff --git a/kadai4/en-ken/kadai4/main_test.go b/kadai4/en-ken/kadai4/main_test.go new file mode 100644 index 0000000..882ff2a --- /dev/null +++ b/kadai4/en-ken/kadai4/main_test.go @@ -0,0 +1,29 @@ +package main_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + main "github.com/gopherdojo/dojo6/kadai4/en-ken/kadai4" +) + +type data struct { + Fortune string `json:"fortune"` +} + +func TestServer(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(main.Handler)) + defer ts.Close() + + res, err := http.Get(ts.URL) + if err != nil { + t.Errorf("%v", err) + } + + var d data + if err = json.NewDecoder(res.Body).Decode(&d); err != nil { + t.Errorf("%v", err) + } +} From ad05f8c0cbe8555cc160ab1a533d783e49c0e4ea Mon Sep 17 00:00:00 2001 From: en-ken Date: Mon, 12 Aug 2019 01:14:41 +0900 Subject: [PATCH 03/15] docs: updated readme. --- kadai4/en-ken/reame.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 kadai4/en-ken/reame.md diff --git a/kadai4/en-ken/reame.md b/kadai4/en-ken/reame.md new file mode 100644 index 0000000..6cd0128 --- /dev/null +++ b/kadai4/en-ken/reame.md @@ -0,0 +1,14 @@ +# 課題4 + +おみくじ API を作りましょう + +- JSON形式でおみくじの結果を返す +- 正月 (1/1-1/3) だけ大吉にする +- ハンドラのテストを書いてみる + +## 使い方 + +```go +go get github.com/gopherdojo/dojo6/kadai4/en-ken/kadai4 +kadai4 +``` From 1de12797d5a62bfefc9a41a38ddec57e1f14156d Mon Sep 17 00:00:00 2001 From: en-ken Date: Mon, 12 Aug 2019 12:22:28 +0900 Subject: [PATCH 04/15] test: added handler test. --- kadai4/en-ken/kadai4/main_test.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/kadai4/en-ken/kadai4/main_test.go b/kadai4/en-ken/kadai4/main_test.go index 882ff2a..0229080 100644 --- a/kadai4/en-ken/kadai4/main_test.go +++ b/kadai4/en-ken/kadai4/main_test.go @@ -19,11 +19,32 @@ func TestServer(t *testing.T) { res, err := http.Get(ts.URL) if err != nil { - t.Errorf("%v", err) + t.Errorf("Response error:%v", err) } var d data if err = json.NewDecoder(res.Body).Decode(&d); err != nil { - t.Errorf("%v", err) + t.Errorf("Data format error:%v", err) + } +} + +func TestHandler(t *testing.T) { + r := httptest.NewRequest("GET", "/", nil) + w := httptest.NewRecorder() + main.SetFortunes([]string{"大凶"}) + + main.Handler(w, r) + + if w.Code != 200 { + t.Errorf("Invalid code: %v", w.Code) + } + if w.Header().Get("Content-Type") != "applicaiton/json; charset=utf-8" { + t.Errorf("Insufficient headers: %v", w.Header()) + } + body := w.Body + dec := json.NewDecoder(body) + var d data + if err := dec.Decode(&d); err != nil || d.Fortune != "大凶" { + t.Errorf("Failed to decode: %v", err) } } From e3e0644203ea6fbf20aa75dcdac47ff1e42e1358 Mon Sep 17 00:00:00 2001 From: en-ken Date: Mon, 12 Aug 2019 12:51:32 +0900 Subject: [PATCH 05/15] refactor: restructured for test. --- kadai4/en-ken/kadai4/export_test.go | 4 ++++ kadai4/en-ken/kadai4/fortune.go | 9 ++++---- kadai4/en-ken/kadai4/main.go | 32 ++++++++++++++++++++--------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/kadai4/en-ken/kadai4/export_test.go b/kadai4/en-ken/kadai4/export_test.go index 9f95ea0..f3e1aff 100644 --- a/kadai4/en-ken/kadai4/export_test.go +++ b/kadai4/en-ken/kadai4/export_test.go @@ -14,3 +14,7 @@ func SetNow(n func() time.Time) { func SetFortunes(f []string) { fortunes = f } + +func SetNewEncoder(n NewEncoder) { + newEncoder = n +} diff --git a/kadai4/en-ken/kadai4/fortune.go b/kadai4/en-ken/kadai4/fortune.go index aab8c95..7eebd47 100644 --- a/kadai4/en-ken/kadai4/fortune.go +++ b/kadai4/en-ken/kadai4/fortune.go @@ -5,14 +5,15 @@ import ( "time" ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - // For testing var now = time.Now var fortunes []string +func init() { + rand.Seed(time.Now().UnixNano()) + fortunes = []string{"大吉", "吉", "中吉", "小吉", "末吉", "凶", "大凶"} +} + func getFortune() string { i := rand.Int() % len(fortunes) f := fortunes[i] diff --git a/kadai4/en-ken/kadai4/main.go b/kadai4/en-ken/kadai4/main.go index 8732308..dd31146 100644 --- a/kadai4/en-ken/kadai4/main.go +++ b/kadai4/en-ken/kadai4/main.go @@ -2,23 +2,40 @@ package main import ( "encoding/json" + "io" "net/http" ) +// Encoder is I/F to json.Encoder +type Encoder interface { + Encode(v interface{}) error +} + +// NewEncoder is I/F to json.NewEncoder +type NewEncoder func(w io.Writer) Encoder + +var newEncoder NewEncoder + func init() { - fortunes = []string{"大吉", "吉", "中吉", "小吉", "末吉", "凶", "大凶"} + // For testing + newEncoder = func(w io.Writer) Encoder { + return json.NewEncoder(w) + } } -type response struct { - Fortune string `json:"fortune"` +func main() { + http.HandleFunc("/", handler) + http.ListenAndServe(":8080", nil) } func handler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "applicaiton/json; charset=utf-8") - enc := json.NewEncoder(w) + enc := newEncoder(w) - resp := &response{ + resp := &struct { + Fortune string `json:"fortune"` + }{ Fortune: getFortune(), } @@ -26,8 +43,3 @@ func handler(w http.ResponseWriter, r *http.Request) { http.Error(w, "Server error", http.StatusInternalServerError) } } - -func main() { - http.HandleFunc("/", handler) - http.ListenAndServe(":8080", nil) -} From 1f888e8576a8f0a40ddc909a5349694c8c04be08 Mon Sep 17 00:00:00 2001 From: en-ken Date: Mon, 12 Aug 2019 13:09:23 +0900 Subject: [PATCH 06/15] test: added internal server error test. --- kadai4/en-ken/kadai4/main_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/kadai4/en-ken/kadai4/main_test.go b/kadai4/en-ken/kadai4/main_test.go index 0229080..1b970bb 100644 --- a/kadai4/en-ken/kadai4/main_test.go +++ b/kadai4/en-ken/kadai4/main_test.go @@ -2,6 +2,8 @@ package main_test import ( "encoding/json" + "fmt" + "io" "net/http" "net/http/httptest" "testing" @@ -28,6 +30,33 @@ func TestServer(t *testing.T) { } } +type EncoderMock struct { +} + +func NewEncoderMock(w io.Writer) main.Encoder { + return &EncoderMock{} +} + +func (enc *EncoderMock) Encode(v interface{}) error { + return fmt.Errorf("Internal server error") +} + +func TestServerInternalServerError(t *testing.T) { + main.SetNewEncoder(NewEncoderMock) + defer main.SetNewEncoder(main.NewEncoder(func(w io.Writer) main.Encoder { + return json.NewEncoder(w) + })) + + ts := httptest.NewServer(http.HandlerFunc(main.Handler)) + defer ts.Close() + + res, _ := http.Get(ts.URL) + if res.StatusCode != 500 { + t.Errorf("Response code error:%v", res.StatusCode) + } + +} + func TestHandler(t *testing.T) { r := httptest.NewRequest("GET", "/", nil) w := httptest.NewRecorder() From 6dd368c1c39c740bc149ca9b950a68c505f5ddbf Mon Sep 17 00:00:00 2001 From: en-ken Date: Wed, 28 Aug 2019 19:04:48 +0900 Subject: [PATCH 07/15] feat: created fileutil and tests. --- kadai3-2/en-ken/fileutil.go | 55 +++++++++++++++++++++++ kadai3-2/en-ken/fileutil_test.go | 76 ++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 kadai3-2/en-ken/fileutil.go create mode 100644 kadai3-2/en-ken/fileutil_test.go diff --git a/kadai3-2/en-ken/fileutil.go b/kadai3-2/en-ken/fileutil.go new file mode 100644 index 0000000..be96592 --- /dev/null +++ b/kadai3-2/en-ken/fileutil.go @@ -0,0 +1,55 @@ +package divdl + +import ( + "fmt" + "io" + "os" + + "github.com/pkg/errors" +) + +// SaveFile saves data as fileName. +func SaveFile(fileName string, data []byte) error { + file, err := os.Create(fileName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to open %v", fileName)) + } + defer file.Close() + + _, err = file.Write(data) + if err != nil { + return errors.WithStack(err) + } + return nil +} + +// MergeFiles merges separeted files. +func MergeFiles(inputFiles []string, outputFileName string) error { + + fw, err := os.Create(outputFileName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to open output file: %v", outputFileName)) + } + defer fw.Close() + + for _, f := range inputFiles { + println(f) + fr, err := os.Open(f) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to open %v", f)) + } + defer fr.Close() + + _, err = io.Copy(fw, fr) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to write data from %v", f)) + } + + if err := os.Remove(f); err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to delete %v", f)) + } + + } + + return nil +} diff --git a/kadai3-2/en-ken/fileutil_test.go b/kadai3-2/en-ken/fileutil_test.go new file mode 100644 index 0000000..8cb737a --- /dev/null +++ b/kadai3-2/en-ken/fileutil_test.go @@ -0,0 +1,76 @@ +package divdl_test + +import ( + "io/ioutil" + "path/filepath" + "testing" + + divdl "github.com/gopherdojo/dojo6/kadai3-2/en-ken" + + "github.com/google/go-cmp/cmp" +) + +var tmpDir string + +func TestMain(m *testing.M) { + tmpDir, _ = ioutil.TempDir("", ".tmp") + m.Run() +} + +func TestSaveFile(t *testing.T) { + file := filepath.Join(tmpDir, "test.txt") + + expected := []byte("foo\nbar\nbaz\n") + divdl.SaveFile(file, expected) + + actual, _ := ioutil.ReadFile(file) + if string(actual) != string(expected) { + t.Errorf("failed. Diff:\n%v", cmp.Diff(actual, expected)) + } +} + +func TestMergeFiles(t *testing.T) { + strA := "0\n1\n2\n3\n4\n" + strB := "5\n6\n7\n8\n9\n" + fileInfo := []struct { + fileName string + data string + }{ + { + fileName: filepath.Join(tmpDir, "merged.txt.0"), + data: strA, + }, + { + fileName: filepath.Join(tmpDir, "merged.txt.1"), + data: strB, + }, + { + fileName: filepath.Join(tmpDir, "merged.txt.2"), + data: strA, + }, + { + fileName: filepath.Join(tmpDir, "merged.txt.3"), + data: strB, + }, + } + + var inputFiles []string + var expected string + for _, fi := range fileInfo { + if err := divdl.SaveFile(fi.fileName, []byte(fi.data)); err != nil { + t.Errorf("SaveFile failed: %v", err) + } + inputFiles = append(inputFiles, fi.fileName) + expected += fi.data + } + + mergedFile := filepath.Join(tmpDir, "merged.txt") + if err := divdl.MergeFiles(inputFiles, mergedFile); err != nil { + t.Errorf("MergeFiles failed: %v", err) + } + + actual, _ := ioutil.ReadFile(mergedFile) + if string(actual) != expected { + t.Errorf("MergeFiles makes invalid data: %v", string(actual)) + } +} From 2e2ce76c4e6c8aea86625ca65f1c1a9cfcc08080 Mon Sep 17 00:00:00 2001 From: en-ken Date: Thu, 29 Aug 2019 23:46:44 +0900 Subject: [PATCH 08/15] feat; added request module and tests. --- kadai3-2/en-ken/export_test.go | 22 ++++ kadai3-2/en-ken/request.go | 89 +++++++++++++++ kadai3-2/en-ken/request_test.go | 188 ++++++++++++++++++++++++++++++++ 3 files changed, 299 insertions(+) create mode 100644 kadai3-2/en-ken/export_test.go create mode 100644 kadai3-2/en-ken/request.go create mode 100644 kadai3-2/en-ken/request_test.go diff --git a/kadai3-2/en-ken/export_test.go b/kadai3-2/en-ken/export_test.go new file mode 100644 index 0000000..fa06aba --- /dev/null +++ b/kadai3-2/en-ken/export_test.go @@ -0,0 +1,22 @@ +package divdl + +var DivideIntoRanges = func(contentLength int64, numOfDivision int) (int, [][]*TestRange) { + n, result := divideIntoRanges(contentLength, numOfDivision) + rngs := make([][]*TestRange, numOfDivision) + for i, rp := range result { + for _, r := range rp { + rngs[i] = append(rngs[i], &TestRange{ + ID: r.id, + From: r.from, + To: r.to, + }) + } + } + return n, rngs +} + +type TestRange struct { + ID int + From int64 + To int64 +} diff --git a/kadai3-2/en-ken/request.go b/kadai3-2/en-ken/request.go new file mode 100644 index 0000000..8b75460 --- /dev/null +++ b/kadai3-2/en-ken/request.go @@ -0,0 +1,89 @@ +package divdl + +import ( + "fmt" + "io/ioutil" + "net/http" + "strconv" + + "github.com/pkg/errors" +) + +// Request wraps downloading proceess. +type Request struct { + url string + contentLength int64 + canAcceptRangeRequest bool +} + +// NewRequest is a constructor of Request. +func NewRequest(url string) (*Request, error) { + resp, err := http.Head(url) + if err != nil { + return nil, errors.WithStack(err) + } + + // Header request before downloading + acceptRanges := resp.Header.Get("Accept-Ranges") + contentLength, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 0) + if err != nil { + return nil, errors.WithStack(err) + } + + return &Request{ + url: url, + contentLength: contentLength, + canAcceptRangeRequest: acceptRanges == "bytes", + }, nil +} + +// DownloadPartially downloads specified part of data from the specified url. +func (r *Request) DownloadPartially(from int64, to int64) ([]byte, error) { + if !r.canAcceptRangeRequest { + return nil, fmt.Errorf("This file cannot download with Range") + } + + // Set Range Header if Range request is accepted. + headers := map[string]string{} + headers["Range"] = fmt.Sprintf("bytes=%d-%d", from, to) + + return r.download(headers) +} + +// Download downloads data from the specified url. +func (r *Request) Download() ([]byte, error) { + return r.download(nil) +} + +func (r *Request) download(extraHeaders map[string]string) ([]byte, error) { + req, err := http.NewRequest("GET", r.url, nil) + if err != nil { + return nil, errors.WithStack(err) + } + for k, v := range extraHeaders { + req.Header.Add(k, v) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.WithStack(err) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll((resp.Body)) + if err != nil { + return nil, errors.WithStack(err) + } + + return body, nil +} + +// CanAcceptRangeRequest is getter of canAcceptRengeRequest +func (r *Request) CanAcceptRangeRequest() bool { + return r.canAcceptRangeRequest +} + +// GetContentLength is getter to content length of data. +func (r *Request) GetContentLength() int64 { + return r.contentLength +} diff --git a/kadai3-2/en-ken/request_test.go b/kadai3-2/en-ken/request_test.go new file mode 100644 index 0000000..f8f3565 --- /dev/null +++ b/kadai3-2/en-ken/request_test.go @@ -0,0 +1,188 @@ +package divdl_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + divdl "github.com/gopherdojo/dojo6/kadai3-2/en-ken" + + "github.com/google/go-cmp/cmp" +) + +func TestNewRequest(t *testing.T) { + cases := []struct { + canHead bool + canAcceptRange bool + }{ + { + canHead: true, + canAcceptRange: true, + }, + { + canHead: true, + canAcceptRange: false, + }, + { + canHead: false, + canAcceptRange: false, + }, + } + + for _, c := range cases { + c := c + + body := []byte("success") + var testHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !c.canHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + if c.canAcceptRange { + w.Header().Add("Accept-Ranges", "bytes") + } + w.Header().Add("Content-Length", fmt.Sprint(len(body))) + fmt.Fprint(w, "") + }) + ts := httptest.NewServer(testHandler) + defer ts.Close() + + req, err := divdl.NewRequest(ts.URL) + if c.canHead { + if err != nil { + t.Errorf("Unexpected result: %v", err) + } + } else { + if err == nil { + t.Error("Http status error epected.") + } + return + } + + if req == nil { + t.Errorf("Unexpected result") + } + if req.CanAcceptRangeRequest() != c.canAcceptRange { + t.Errorf("Unexpected result") + } + } +} + +func TestDownload(t *testing.T) { + cases := []struct { + canGet bool + }{ + { + canGet: true, + }, + { + canGet: false, + }, + } + + for _, c := range cases { + c := c + + bodyStr := fmt.Sprintf("%v%v%v%v%v", + "0000000000", + "1111111111", + "2222222222", + "3333333333", + "4444444444", + ) + var testHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Add("Content-Length", fmt.Sprint(len(bodyStr))) + fmt.Print(w, "") + return + } + + if c.canGet { + fmt.Fprintf(w, "%v", bodyStr) + } else { + w.WriteHeader(http.StatusNotFound) + fmt.Print(w, "") + } + }) + ts := httptest.NewServer(testHandler) + defer ts.Close() + + req, _ := divdl.NewRequest(ts.URL) + data, err := req.Download() + if !c.canGet { + if err == nil { + t.Error("Unexpected http status") + } + return + } + + if !cmp.Equal([]byte(bodyStr), data) { + t.Errorf("Unexpected response: Diff\n%v", cmp.Diff(bodyStr, data)) + } + } +} + +func TestDownloadPartially(t *testing.T) { + cases := []struct { + canGet bool + from int64 + to int64 + rngStr string + expected string + }{ + { + canGet: true, + from: 0, + to: 10, + rngStr: "bytes=0-10", + expected: "0000000000", + }, + { + canGet: false, + }, + } + + for _, c := range cases { + c := c + + bodyStr := fmt.Sprintf("%v%v%v%v%v", + "0000000000", + "1111111111", + "2222222222", + "3333333333", + "4444444444", + ) + var testHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Add("Accept-Ranges", "bytes") + w.Header().Add("Content-Length", fmt.Sprint(len(bodyStr))) + fmt.Print(w, "") + } + if c.canGet { + if c.rngStr == r.Header.Get("Range") { + fmt.Fprint(w, "0000000000") + } + } else { + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, "") + } + + }) + ts := httptest.NewServer(testHandler) + defer ts.Close() + + req, _ := divdl.NewRequest(ts.URL) + actual, err := req.DownloadPartially(c.from, c.to) + if !c.canGet { + if err == nil { + t.Error("Unexpected http status") + } + } + + if !cmp.Equal([]byte(c.expected), actual) { + t.Errorf("Unexpected response: Diff\n%v", cmp.Diff(c.expected, actual)) + } + } +} From 667906c62e0e19db1c798815fe551c89b6090466 Mon Sep 17 00:00:00 2001 From: en-ken Date: Fri, 30 Aug 2019 01:38:01 +0900 Subject: [PATCH 09/15] feat: added goroutine management logics. --- kadai3-2/en-ken/div-dl/main.go | 43 +++++++++++++ kadai3-2/en-ken/dl-manager.go | 107 +++++++++++++++++++++++++++++++++ kadai3-2/en-ken/fileutil.go | 3 +- 3 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 kadai3-2/en-ken/div-dl/main.go create mode 100644 kadai3-2/en-ken/dl-manager.go diff --git a/kadai3-2/en-ken/div-dl/main.go b/kadai3-2/en-ken/div-dl/main.go new file mode 100644 index 0000000..4ec025d --- /dev/null +++ b/kadai3-2/en-ken/div-dl/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path" + + divdl "github.com/gopherdojo/dojo6/kadai3-2/en-ken" + + "github.com/pkg/errors" +) + +// Do is I/F to divdl.Do +type Do func(url string, fileName string, numOfDivision int) error + +func outputError(err error) { + os.Stderr.Write([]byte(fmt.Sprintf("%v", errors.WithStack(err)))) +} + +func main() { + var ( + fileName string + numOfDivision int + ) + + flag.StringVar(&fileName, "o", "[remote-name]", "output file name") + flag.IntVar(&numOfDivision, "n", 5, "number of downloading in parallel") + flag.Parse() + url := flag.Arg(0) + // Set remote name as fileName + if fileName == "[remote-name]" { + fileName = path.Base(url) + } + + err := Do(divdl.Do)(url, fileName, numOfDivision) + if err != nil { + outputError(err) + return + } + + os.Stdout.Write([]byte("Downloading done.\n")) +} diff --git a/kadai3-2/en-ken/dl-manager.go b/kadai3-2/en-ken/dl-manager.go new file mode 100644 index 0000000..0189a0d --- /dev/null +++ b/kadai3-2/en-ken/dl-manager.go @@ -0,0 +1,107 @@ +package divdl + +import ( + "fmt" + + "golang.org/x/sync/errgroup" + + "github.com/pkg/errors" +) + +// DlRange expresses range of Range request +type DlRange struct { + id int + from int64 + to int64 +} + +// MaxRangeSize is limit size of range +const MaxRangeSize = 1024 * 1024 //1MB + +func divideIntoRanges(contentLength int64, numOfDivision int) (numOfRanges int, rngs [][]*DlRange) { + rngs = make([][]*DlRange, numOfDivision) + + rngSize := (contentLength + int64(numOfDivision)) / int64(numOfDivision) + if MaxRangeSize < rngSize { + rngSize = MaxRangeSize + } + + for j, pos := 0, int64(0); pos < contentLength; j++ { + for i := 0; i < numOfDivision; i++ { + id := j*numOfDivision + i + numOfRanges = id + 1 + // Last range + if contentLength-pos < rngSize { + rngs[i] = append(rngs[i], &DlRange{ + id: id, + from: int64(id) * rngSize, + to: contentLength - 1, + }) + pos = contentLength + break + } + + rngs[i] = append(rngs[i], &DlRange{ + id: id, + from: int64(id) * rngSize, + to: int64(id+1)*rngSize - 1, + }) + pos += rngSize + } + } + return +} + +// Do manages separately downloading. +func Do(url string, fileName string, numOfDivision int) error { + req, err := NewRequest(url) + if err != nil { + return errors.WithStack(err) + } + + // Range request is not accepted + if !req.CanAcceptRangeRequest() { + data, err := req.Download() + if err != nil { + return err + } + + return SaveFile(fileName, data) + } + + // Range request is accepted + n, rngs := divideIntoRanges(req.GetContentLength(), numOfDivision) + + var g errgroup.Group + for _, rSet := range rngs { + rSet := rSet + g.Go(func() error { + for _, r := range rSet { + tmpFileName := createPartialFileName(fileName, r.id) + data, err := req.DownloadPartially(r.from, r.to) + if err != nil { + return err + } + if err := SaveFile(tmpFileName, data); err != nil { + return err + } + fmt.Printf("%v saved\n", tmpFileName) + } + return nil + }) + } + + if err := g.Wait(); err != nil { + return errors.WithStack(err) + } + + files := make([]string, 0) + for i := 0; i < n; i++ { + files = append(files, createPartialFileName(fileName, i)) + } + return MergeFiles(files, fileName) +} + +func createPartialFileName(fileName string, suffix int) string { + return fmt.Sprintf("%v.%v", fileName, suffix) +} diff --git a/kadai3-2/en-ken/fileutil.go b/kadai3-2/en-ken/fileutil.go index be96592..9e0f291 100644 --- a/kadai3-2/en-ken/fileutil.go +++ b/kadai3-2/en-ken/fileutil.go @@ -33,10 +33,9 @@ func MergeFiles(inputFiles []string, outputFileName string) error { defer fw.Close() for _, f := range inputFiles { - println(f) fr, err := os.Open(f) if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to open %v", f)) + return errors.Wrap(err, fmt.Sprintf("Failed to open [%v]", f)) } defer fr.Close() From 9f6fc7eec87b02c7462602bd7ca200130f421954 Mon Sep 17 00:00:00 2001 From: en-ken Date: Sun, 1 Sep 2019 10:33:23 +0900 Subject: [PATCH 10/15] test: fixed request test cases. --- kadai3-2/en-ken/request_test.go | 131 ++++++++++++-------------------- 1 file changed, 50 insertions(+), 81 deletions(-) diff --git a/kadai3-2/en-ken/request_test.go b/kadai3-2/en-ken/request_test.go index f8f3565..2a9f06b 100644 --- a/kadai3-2/en-ken/request_test.go +++ b/kadai3-2/en-ken/request_test.go @@ -71,73 +71,34 @@ func TestNewRequest(t *testing.T) { } func TestDownload(t *testing.T) { + bodyStr := fmt.Sprintf("%v%v%v%v%v", + "0000000000", + "1111111111", + "2222222222", + "3333333333", + "4444444444", + ) + cases := []struct { - canGet bool + canGet bool + canGetRange bool + from int64 + to int64 + rngStr string + expected []byte }{ { - canGet: true, + canGet: true, + canGetRange: true, + from: 10, + to: 19, + rngStr: "bytes=10-19", + expected: []byte(bodyStr[10:20]), }, { - canGet: false, - }, - } - - for _, c := range cases { - c := c - - bodyStr := fmt.Sprintf("%v%v%v%v%v", - "0000000000", - "1111111111", - "2222222222", - "3333333333", - "4444444444", - ) - var testHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == "HEAD" { - w.Header().Add("Content-Length", fmt.Sprint(len(bodyStr))) - fmt.Print(w, "") - return - } - - if c.canGet { - fmt.Fprintf(w, "%v", bodyStr) - } else { - w.WriteHeader(http.StatusNotFound) - fmt.Print(w, "") - } - }) - ts := httptest.NewServer(testHandler) - defer ts.Close() - - req, _ := divdl.NewRequest(ts.URL) - data, err := req.Download() - if !c.canGet { - if err == nil { - t.Error("Unexpected http status") - } - return - } - - if !cmp.Equal([]byte(bodyStr), data) { - t.Errorf("Unexpected response: Diff\n%v", cmp.Diff(bodyStr, data)) - } - } -} - -func TestDownloadPartially(t *testing.T) { - cases := []struct { - canGet bool - from int64 - to int64 - rngStr string - expected string - }{ - { - canGet: true, - from: 0, - to: 10, - rngStr: "bytes=0-10", - expected: "0000000000", + canGet: true, + canGetRange: false, + expected: []byte(bodyStr), }, { canGet: false, @@ -146,42 +107,50 @@ func TestDownloadPartially(t *testing.T) { for _, c := range cases { c := c + bodyStr := bodyStr - bodyStr := fmt.Sprintf("%v%v%v%v%v", - "0000000000", - "1111111111", - "2222222222", - "3333333333", - "4444444444", - ) var testHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == "HEAD" { - w.Header().Add("Accept-Ranges", "bytes") + switch r.Method { + case "HEAD": + if c.canGetRange { + w.Header().Add("Accept-Ranges", "bytes") + } w.Header().Add("Content-Length", fmt.Sprint(len(bodyStr))) fmt.Print(w, "") - } - if c.canGet { - if c.rngStr == r.Header.Get("Range") { - fmt.Fprint(w, "0000000000") + case "GET": + if c.canGet { + if r.Header.Get("Range") == c.rngStr { + fmt.Fprint(w, bodyStr[c.from:c.to+1]) + } else { + fmt.Fprint(w, bodyStr) + } + } else { + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, "") } - } else { - w.WriteHeader(http.StatusNotFound) - fmt.Fprint(w, "") } - }) ts := httptest.NewServer(testHandler) defer ts.Close() req, _ := divdl.NewRequest(ts.URL) - actual, err := req.DownloadPartially(c.from, c.to) + var ( + actual []byte + err error + ) + if req.CanAcceptRangeRequest() { + actual, err = req.DownloadPartially(c.from, c.to) + } else { + actual, err = req.Download() + } + if !c.canGet { if err == nil { t.Error("Unexpected http status") } } - if !cmp.Equal([]byte(c.expected), actual) { + if !cmp.Equal(c.expected, actual) { t.Errorf("Unexpected response: Diff\n%v", cmp.Diff(c.expected, actual)) } } From 1542f1adfabb7614f921cbfdd41d265e9111d0c2 Mon Sep 17 00:00:00 2001 From: en-ken Date: Sun, 1 Sep 2019 10:56:58 +0900 Subject: [PATCH 11/15] fix: fixed bugs. --- kadai3-2/en-ken/request.go | 3 +++ kadai3-2/en-ken/request_test.go | 41 ++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/kadai3-2/en-ken/request.go b/kadai3-2/en-ken/request.go index 8b75460..886c336 100644 --- a/kadai3-2/en-ken/request.go +++ b/kadai3-2/en-ken/request.go @@ -65,6 +65,9 @@ func (r *Request) download(extraHeaders map[string]string) ([]byte, error) { } resp, err := http.DefaultClient.Do(req) + if resp.Status[:1] != "2" { + return nil, errors.Errorf("Faild to request: %v", resp.StatusCode) + } if err != nil { return nil, errors.WithStack(err) } diff --git a/kadai3-2/en-ken/request_test.go b/kadai3-2/en-ken/request_test.go index 2a9f06b..079b691 100644 --- a/kadai3-2/en-ken/request_test.go +++ b/kadai3-2/en-ken/request_test.go @@ -105,7 +105,7 @@ func TestDownload(t *testing.T) { }, } - for _, c := range cases { + for i, c := range cases { c := c bodyStr := bodyStr @@ -119,7 +119,8 @@ func TestDownload(t *testing.T) { fmt.Print(w, "") case "GET": if c.canGet { - if r.Header.Get("Range") == c.rngStr { + rng := r.Header.Get("Range") + if rng != "" && rng == c.rngStr { fmt.Fprint(w, bodyStr[c.from:c.to+1]) } else { fmt.Fprint(w, bodyStr) @@ -133,25 +134,27 @@ func TestDownload(t *testing.T) { ts := httptest.NewServer(testHandler) defer ts.Close() - req, _ := divdl.NewRequest(ts.URL) - var ( - actual []byte - err error - ) - if req.CanAcceptRangeRequest() { - actual, err = req.DownloadPartially(c.from, c.to) - } else { - actual, err = req.Download() - } + t.Run(fmt.Sprintf("case %v", i), func(t *testing.T) { + req, _ := divdl.NewRequest(ts.URL) + var ( + actual []byte + err error + ) + if req.CanAcceptRangeRequest() { + actual, err = req.DownloadPartially(c.from, c.to) + } else { + actual, err = req.Download() + } - if !c.canGet { - if err == nil { - t.Error("Unexpected http status") + if !c.canGet { + if err == nil { + t.Error("Unexpected http status") + } } - } - if !cmp.Equal(c.expected, actual) { - t.Errorf("Unexpected response: Diff\n%v", cmp.Diff(c.expected, actual)) - } + if !cmp.Equal(c.expected, actual) { + t.Errorf("Unexpected response: Diff\n%v", cmp.Diff(c.expected, actual)) + } + }) } } From 8353167df630eebd7b6ece5df6864b30e005bae5 Mon Sep 17 00:00:00 2001 From: en-ken Date: Sun, 1 Sep 2019 11:31:38 +0900 Subject: [PATCH 12/15] refactor: changed paths --- go.mod | 9 ++ go.sum | 6 + kadai3-2/en-ken/Readme.md | 22 +++ kadai3-2/en-ken/dl-manager.go | 23 +-- kadai3-2/en-ken/dl-manager_test.go | 151 +++++++++++++++++++ kadai3-2/en-ken/{div-dl => dl-mgr}/main.go | 0 kadai3-2/en-ken/export_test.go | 2 + kadai3-2/en-ken/{ => utils}/fileutil.go | 2 +- kadai3-2/en-ken/{ => utils}/fileutil_test.go | 10 +- kadai3-2/en-ken/{ => utils}/request.go | 2 +- kadai3-2/en-ken/{ => utils}/request_test.go | 8 +- 11 files changed, 215 insertions(+), 20 deletions(-) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 kadai3-2/en-ken/Readme.md create mode 100644 kadai3-2/en-ken/dl-manager_test.go rename kadai3-2/en-ken/{div-dl => dl-mgr}/main.go (100%) rename kadai3-2/en-ken/{ => utils}/fileutil.go (98%) rename kadai3-2/en-ken/{ => utils}/fileutil_test.go (85%) rename kadai3-2/en-ken/{ => utils}/request.go (99%) rename kadai3-2/en-ken/{ => utils}/request_test.go (95%) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..08c11d6 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/gopherdojo/dojo6 + +go 1.12 + +require ( + github.com/google/go-cmp v0.3.1 + github.com/pkg/errors v0.8.1 + golang.org/x/sync v0.0.0-20190423024810-112230192c58 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..968da45 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/kadai3-2/en-ken/Readme.md b/kadai3-2/en-ken/Readme.md new file mode 100644 index 0000000..3a71f26 --- /dev/null +++ b/kadai3-2/en-ken/Readme.md @@ -0,0 +1,22 @@ +# 課題3-2 + +## 分割ダウンローダを作ろう + +- Rangeアクセスを用いる +- いくつかのゴルーチンでダウンロードしてマージする +- エラー処理を工夫する + - golang.org/x/sync/errgroupパッケージなどを使ってみる +- キャンセルが発生した場合の実装を行う + +## やること + +1. HEADリクエストしてAccept-Rangesヘッダの有無確認 +1. Accept-Rangesがあった場合、 + 1. 任意の分割数に応じて、goroutineごとの割当範囲を決めて、各goroutineでRange GET + 1. 部分ファイルに出力 +1. 全部ダウンロードできたら、ファイルをマージして部分ファイル削除 + +## 工夫した点 + +- 1つのgoroutineがダウンロードするデータが1MBを超えた場合、1MBごとにファイル出力するようにした。 +- すでにダウンロード済のデータがあった場合(そのパートのファイルがあった場合)、それを再利用する。 diff --git a/kadai3-2/en-ken/dl-manager.go b/kadai3-2/en-ken/dl-manager.go index 0189a0d..a8186f4 100644 --- a/kadai3-2/en-ken/dl-manager.go +++ b/kadai3-2/en-ken/dl-manager.go @@ -5,6 +5,7 @@ import ( "golang.org/x/sync/errgroup" + "github.com/gopherdojo/dojo6/kadai3-2/en-ken/utils" "github.com/pkg/errors" ) @@ -15,15 +16,19 @@ type DlRange struct { to int64 } -// MaxRangeSize is limit size of range -const MaxRangeSize = 1024 * 1024 //1MB +const maxRangeSize = 1024 * 1024 //1MB func divideIntoRanges(contentLength int64, numOfDivision int) (numOfRanges int, rngs [][]*DlRange) { rngs = make([][]*DlRange, numOfDivision) - rngSize := (contentLength + int64(numOfDivision)) / int64(numOfDivision) - if MaxRangeSize < rngSize { - rngSize = MaxRangeSize + var rngSize int64 + if contentLength%int64(numOfDivision) == 0 { + rngSize = contentLength / int64(numOfDivision) + } else { + rngSize = (contentLength + int64(numOfDivision)) / int64(numOfDivision) + } + if maxRangeSize < rngSize { + rngSize = maxRangeSize } for j, pos := 0, int64(0); pos < contentLength; j++ { @@ -54,7 +59,7 @@ func divideIntoRanges(contentLength int64, numOfDivision int) (numOfRanges int, // Do manages separately downloading. func Do(url string, fileName string, numOfDivision int) error { - req, err := NewRequest(url) + req, err := utils.NewRequest(url) if err != nil { return errors.WithStack(err) } @@ -66,7 +71,7 @@ func Do(url string, fileName string, numOfDivision int) error { return err } - return SaveFile(fileName, data) + return utils.SaveFile(fileName, data) } // Range request is accepted @@ -82,7 +87,7 @@ func Do(url string, fileName string, numOfDivision int) error { if err != nil { return err } - if err := SaveFile(tmpFileName, data); err != nil { + if err := utils.SaveFile(tmpFileName, data); err != nil { return err } fmt.Printf("%v saved\n", tmpFileName) @@ -99,7 +104,7 @@ func Do(url string, fileName string, numOfDivision int) error { for i := 0; i < n; i++ { files = append(files, createPartialFileName(fileName, i)) } - return MergeFiles(files, fileName) + return utils.MergeFiles(files, fileName) } func createPartialFileName(fileName string, suffix int) string { diff --git a/kadai3-2/en-ken/dl-manager_test.go b/kadai3-2/en-ken/dl-manager_test.go new file mode 100644 index 0000000..a49ed2e --- /dev/null +++ b/kadai3-2/en-ken/dl-manager_test.go @@ -0,0 +1,151 @@ +package divdl_test + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + divdl "github.com/gopherdojo/dojo6/kadai3-2/en-ken" +) + +func TestDivideIntoRanges(t *testing.T) { + + type testCase struct { + contentLength int64 + num int + expected [][]*divdl.TestRange + expectedNum int + } + + cases := []testCase{ + { + contentLength: 100, + num: 2, + expected: [][]*divdl.TestRange{ + { + &divdl.TestRange{ID: 0, From: 0, To: 49}, + }, + { + &divdl.TestRange{ID: 1, From: 50, To: 99}, + }, + }, + expectedNum: 2, + }, + { + contentLength: 1000, + num: 5, + expected: [][]*divdl.TestRange{ + { + &divdl.TestRange{ID: 0, From: 0, To: 199}, + }, + { + &divdl.TestRange{ID: 1, From: 200, To: 399}, + }, + { + &divdl.TestRange{ID: 2, From: 400, To: 599}, + }, + { + &divdl.TestRange{ID: 3, From: 600, To: 799}, + }, + { + &divdl.TestRange{ID: 4, From: 800, To: 999}, + }, + }, + expectedNum: 5, + }, + { + contentLength: 1001, + num: 5, + expected: [][]*divdl.TestRange{ + { + &divdl.TestRange{ID: 0, From: 0, To: 200}, + }, + { + &divdl.TestRange{ID: 1, From: 201, To: 401}, + }, + { + &divdl.TestRange{ID: 2, From: 402, To: 602}, + }, + { + &divdl.TestRange{ID: 3, From: 603, To: 803}, + }, + { + &divdl.TestRange{ID: 4, From: 804, To: 1000}, + }, + }, + expectedNum: 5, + }, + { + contentLength: 1005, + num: 5, + expected: [][]*divdl.TestRange{ + { + &divdl.TestRange{ID: 0, From: 0, To: 200}, + }, + { + &divdl.TestRange{ID: 1, From: 201, To: 401}, + }, + { + &divdl.TestRange{ID: 2, From: 402, To: 602}, + }, + { + &divdl.TestRange{ID: 3, From: 603, To: 803}, + }, + { + &divdl.TestRange{ID: 4, From: 804, To: 1004}, + }, + }, + expectedNum: 5, + }, + { + contentLength: divdl.MaxRangeSize*8 - 10, + num: 5, + expected: [][]*divdl.TestRange{ + { + &divdl.TestRange{ID: 0, From: 0, To: divdl.MaxRangeSize - 1}, + &divdl.TestRange{ID: 5, From: divdl.MaxRangeSize * 5, To: divdl.MaxRangeSize*6 - 1}, + }, + { + &divdl.TestRange{ID: 1, From: divdl.MaxRangeSize, To: divdl.MaxRangeSize*2 - 1}, + &divdl.TestRange{ID: 6, From: divdl.MaxRangeSize * 6, To: divdl.MaxRangeSize * 7}, + }, + { + &divdl.TestRange{ID: 2, From: divdl.MaxRangeSize * 2, To: divdl.MaxRangeSize*3 - 1}, + &divdl.TestRange{ID: 7, From: divdl.MaxRangeSize * 7, To: divdl.MaxRangeSize*8 - 11}, + }, + { + &divdl.TestRange{ID: 3, From: divdl.MaxRangeSize * 3, To: divdl.MaxRangeSize*4 - 1}, + }, + { + &divdl.TestRange{ID: 4, From: divdl.MaxRangeSize * 4, To: divdl.MaxRangeSize*5 - 1}, + }, + }, + expectedNum: 8, + }, + { + contentLength: divdl.MaxRangeSize * 3, + num: 2, + expected: [][]*divdl.TestRange{ + { + &divdl.TestRange{ID: 0, From: 0, To: divdl.MaxRangeSize - 1}, + &divdl.TestRange{ID: 2, From: divdl.MaxRangeSize * 2, To: divdl.MaxRangeSize*3 - 1}, + }, + { + &divdl.TestRange{ID: 1, From: divdl.MaxRangeSize, To: divdl.MaxRangeSize*2 - 1}, + }, + }, + expectedNum: 3, + }, + } + + for i, c := range cases { + c := c + t.Run(fmt.Sprintf("case %v", i), func(t *testing.T) { + n, actual := divdl.DivideIntoRanges(c.contentLength, c.num) + + if !cmp.Equal(actual, c.expected) || n != c.expectedNum { + t.Errorf("failed. Diff:\n%v", cmp.Diff(actual, c.expected)) + } + }) + } +} diff --git a/kadai3-2/en-ken/div-dl/main.go b/kadai3-2/en-ken/dl-mgr/main.go similarity index 100% rename from kadai3-2/en-ken/div-dl/main.go rename to kadai3-2/en-ken/dl-mgr/main.go diff --git a/kadai3-2/en-ken/export_test.go b/kadai3-2/en-ken/export_test.go index fa06aba..691e9e2 100644 --- a/kadai3-2/en-ken/export_test.go +++ b/kadai3-2/en-ken/export_test.go @@ -20,3 +20,5 @@ type TestRange struct { From int64 To int64 } + +const MaxRangeSize = maxRangeSize diff --git a/kadai3-2/en-ken/fileutil.go b/kadai3-2/en-ken/utils/fileutil.go similarity index 98% rename from kadai3-2/en-ken/fileutil.go rename to kadai3-2/en-ken/utils/fileutil.go index 9e0f291..138017c 100644 --- a/kadai3-2/en-ken/fileutil.go +++ b/kadai3-2/en-ken/utils/fileutil.go @@ -1,4 +1,4 @@ -package divdl +package utils import ( "fmt" diff --git a/kadai3-2/en-ken/fileutil_test.go b/kadai3-2/en-ken/utils/fileutil_test.go similarity index 85% rename from kadai3-2/en-ken/fileutil_test.go rename to kadai3-2/en-ken/utils/fileutil_test.go index 8cb737a..b9c5558 100644 --- a/kadai3-2/en-ken/fileutil_test.go +++ b/kadai3-2/en-ken/utils/fileutil_test.go @@ -1,11 +1,11 @@ -package divdl_test +package utils_test import ( "io/ioutil" "path/filepath" "testing" - divdl "github.com/gopherdojo/dojo6/kadai3-2/en-ken" + "github.com/gopherdojo/dojo6/kadai3-2/en-ken/utils" "github.com/google/go-cmp/cmp" ) @@ -21,7 +21,7 @@ func TestSaveFile(t *testing.T) { file := filepath.Join(tmpDir, "test.txt") expected := []byte("foo\nbar\nbaz\n") - divdl.SaveFile(file, expected) + utils.SaveFile(file, expected) actual, _ := ioutil.ReadFile(file) if string(actual) != string(expected) { @@ -57,7 +57,7 @@ func TestMergeFiles(t *testing.T) { var inputFiles []string var expected string for _, fi := range fileInfo { - if err := divdl.SaveFile(fi.fileName, []byte(fi.data)); err != nil { + if err := utils.SaveFile(fi.fileName, []byte(fi.data)); err != nil { t.Errorf("SaveFile failed: %v", err) } inputFiles = append(inputFiles, fi.fileName) @@ -65,7 +65,7 @@ func TestMergeFiles(t *testing.T) { } mergedFile := filepath.Join(tmpDir, "merged.txt") - if err := divdl.MergeFiles(inputFiles, mergedFile); err != nil { + if err := utils.MergeFiles(inputFiles, mergedFile); err != nil { t.Errorf("MergeFiles failed: %v", err) } diff --git a/kadai3-2/en-ken/request.go b/kadai3-2/en-ken/utils/request.go similarity index 99% rename from kadai3-2/en-ken/request.go rename to kadai3-2/en-ken/utils/request.go index 886c336..90e536d 100644 --- a/kadai3-2/en-ken/request.go +++ b/kadai3-2/en-ken/utils/request.go @@ -1,4 +1,4 @@ -package divdl +package utils import ( "fmt" diff --git a/kadai3-2/en-ken/request_test.go b/kadai3-2/en-ken/utils/request_test.go similarity index 95% rename from kadai3-2/en-ken/request_test.go rename to kadai3-2/en-ken/utils/request_test.go index 079b691..6130116 100644 --- a/kadai3-2/en-ken/request_test.go +++ b/kadai3-2/en-ken/utils/request_test.go @@ -1,4 +1,4 @@ -package divdl_test +package utils import ( "fmt" @@ -6,7 +6,7 @@ import ( "net/http/httptest" "testing" - divdl "github.com/gopherdojo/dojo6/kadai3-2/en-ken" + "github.com/gopherdojo/dojo6/kadai3-2/en-ken/utils" "github.com/google/go-cmp/cmp" ) @@ -49,7 +49,7 @@ func TestNewRequest(t *testing.T) { ts := httptest.NewServer(testHandler) defer ts.Close() - req, err := divdl.NewRequest(ts.URL) + req, err := utils.NewRequest(ts.URL) if c.canHead { if err != nil { t.Errorf("Unexpected result: %v", err) @@ -135,7 +135,7 @@ func TestDownload(t *testing.T) { defer ts.Close() t.Run(fmt.Sprintf("case %v", i), func(t *testing.T) { - req, _ := divdl.NewRequest(ts.URL) + req, _ := utils.NewRequest(ts.URL) var ( actual []byte err error From b7ec3833b903672d74a3a1864f5ac6fab3acc842 Mon Sep 17 00:00:00 2001 From: en-ken Date: Sun, 1 Sep 2019 11:55:21 +0900 Subject: [PATCH 13/15] fix: fixed a bug --- kadai3-2/en-ken/dl-manager.go | 14 +++++++++++++- kadai3-2/en-ken/dl-manager_test.go | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/kadai3-2/en-ken/dl-manager.go b/kadai3-2/en-ken/dl-manager.go index a8186f4..3c6d4fd 100644 --- a/kadai3-2/en-ken/dl-manager.go +++ b/kadai3-2/en-ken/dl-manager.go @@ -2,6 +2,7 @@ package divdl import ( "fmt" + "os" "golang.org/x/sync/errgroup" @@ -32,7 +33,7 @@ func divideIntoRanges(contentLength int64, numOfDivision int) (numOfRanges int, } for j, pos := 0, int64(0); pos < contentLength; j++ { - for i := 0; i < numOfDivision; i++ { + for i := 0; i < numOfDivision && pos < contentLength; i++ { id := j*numOfDivision + i numOfRanges = id + 1 // Last range @@ -83,6 +84,12 @@ func Do(url string, fileName string, numOfDivision int) error { g.Go(func() error { for _, r := range rSet { tmpFileName := createPartialFileName(fileName, r.id) + + // pass downloading if tmpFileName exists. + if fileExists(tmpFileName) { + continue + } + data, err := req.DownloadPartially(r.from, r.to) if err != nil { return err @@ -110,3 +117,8 @@ func Do(url string, fileName string, numOfDivision int) error { func createPartialFileName(fileName string, suffix int) string { return fmt.Sprintf("%v.%v", fileName, suffix) } + +func fileExists(fileName string) bool { + _, err := os.Stat(fileName) + return err == nil +} diff --git a/kadai3-2/en-ken/dl-manager_test.go b/kadai3-2/en-ken/dl-manager_test.go index a49ed2e..a6cec67 100644 --- a/kadai3-2/en-ken/dl-manager_test.go +++ b/kadai3-2/en-ken/dl-manager_test.go @@ -107,11 +107,11 @@ func TestDivideIntoRanges(t *testing.T) { }, { &divdl.TestRange{ID: 1, From: divdl.MaxRangeSize, To: divdl.MaxRangeSize*2 - 1}, - &divdl.TestRange{ID: 6, From: divdl.MaxRangeSize * 6, To: divdl.MaxRangeSize * 7}, + &divdl.TestRange{ID: 6, From: divdl.MaxRangeSize * 6, To: divdl.MaxRangeSize*7 - 1}, }, { &divdl.TestRange{ID: 2, From: divdl.MaxRangeSize * 2, To: divdl.MaxRangeSize*3 - 1}, - &divdl.TestRange{ID: 7, From: divdl.MaxRangeSize * 7, To: divdl.MaxRangeSize*8 - 11}, + &divdl.TestRange{ID: 7, From: divdl.MaxRangeSize * 7, To: divdl.MaxRangeSize*8 - 10 - 1}, }, { &divdl.TestRange{ID: 3, From: divdl.MaxRangeSize * 3, To: divdl.MaxRangeSize*4 - 1}, From 0e797c9e4f6978005dcf6cf234f0c1a054808199 Mon Sep 17 00:00:00 2001 From: en-ken Date: Sun, 1 Sep 2019 12:59:27 +0900 Subject: [PATCH 14/15] tests: added dl-manager tests. --- kadai3-2/en-ken/dl-manager.go | 10 +-- kadai3-2/en-ken/dl-manager_test.go | 108 +++++++++++++++++++++++++++++ kadai3-2/en-ken/export_test.go | 6 +- 3 files changed, 118 insertions(+), 6 deletions(-) diff --git a/kadai3-2/en-ken/dl-manager.go b/kadai3-2/en-ken/dl-manager.go index 3c6d4fd..83d02de 100644 --- a/kadai3-2/en-ken/dl-manager.go +++ b/kadai3-2/en-ken/dl-manager.go @@ -17,7 +17,7 @@ type DlRange struct { to int64 } -const maxRangeSize = 1024 * 1024 //1MB +var maxRangeSize = int64(1024 * 1024) //1MB func divideIntoRanges(contentLength int64, numOfDivision int) (numOfRanges int, rngs [][]*DlRange) { rngs = make([][]*DlRange, numOfDivision) @@ -79,13 +79,13 @@ func Do(url string, fileName string, numOfDivision int) error { n, rngs := divideIntoRanges(req.GetContentLength(), numOfDivision) var g errgroup.Group - for _, rSet := range rngs { - rSet := rSet + for _, rList := range rngs { + rList := rList g.Go(func() error { - for _, r := range rSet { + for _, r := range rList { tmpFileName := createPartialFileName(fileName, r.id) - // pass downloading if tmpFileName exists. + // Pass downloading if tmpFileName exists. if fileExists(tmpFileName) { continue } diff --git a/kadai3-2/en-ken/dl-manager_test.go b/kadai3-2/en-ken/dl-manager_test.go index a6cec67..7af553d 100644 --- a/kadai3-2/en-ken/dl-manager_test.go +++ b/kadai3-2/en-ken/dl-manager_test.go @@ -2,6 +2,12 @@ package divdl_test import ( "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "regexp" + "strconv" "testing" "github.com/google/go-cmp/cmp" @@ -149,3 +155,105 @@ func TestDivideIntoRanges(t *testing.T) { }) } } + +func TestDo(t *testing.T) { + type testCase struct { + newRequestError bool + downloadError bool + canGetRange bool + } + + cases := []testCase{ + { + canGetRange: true, + }, + { + canGetRange: false, + }, + { + newRequestError: true, + }, + { + canGetRange: true, + downloadError: true, + }, + { + canGetRange: false, + downloadError: true, + }, + } + + bodyStr := fmt.Sprintf("%v%v%v%v%v", + "0000000000", + "1111111111", + "2222222222", + "3333333333", + "4444444444", + ) + + divdl.SetMaxRangeSize(10) + tmpDir, _ := ioutil.TempDir("", ".tmp") + fileName := filepath.Join(tmpDir, "test.txt") + + for i, c := range cases { + c := c + + var testHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "HEAD": + if c.newRequestError { + w.WriteHeader(http.StatusMethodNotAllowed) + fmt.Fprint(w, "") + return + } + + if c.canGetRange { + w.Header().Add("Accept-Ranges", "bytes") + } + w.Header().Add("Content-Length", fmt.Sprint(len(bodyStr))) + fmt.Print(w, "") + case "GET": + if c.downloadError { + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, "") + return + } + + rngStr := r.Header.Get("Range") + if rngStr != "" { + reg := regexp.MustCompile(`\d+`) + rngs := reg.FindAllString(rngStr, -1) + from, _ := strconv.ParseInt(rngs[0], 10, 0) + to, _ := strconv.ParseInt(rngs[1], 10, 0) + fmt.Fprint(w, bodyStr[from:to+1]) + } else { + fmt.Fprint(w, bodyStr) + } + } + }) + ts := httptest.NewServer(testHandler) + defer ts.Close() + + t.Run(fmt.Sprintf("case %v", i), func(t *testing.T) { + err := divdl.Do(ts.URL, fileName, 3) + if err != nil { + if !c.newRequestError && + !c.downloadError { + t.Errorf("Unexpected result: %v", err) + } + } else { + if c.newRequestError || + c.downloadError { + t.Errorf("Unexpected result: newRequestError->%v, downloadError->%v", + c.newRequestError, c.downloadError) + } + } + + actual, _ := ioutil.ReadFile(fileName) + if string(actual) != bodyStr { + t.Errorf("Downloading file is invalid data: %v", string(actual)) + } + + }) + } +} diff --git a/kadai3-2/en-ken/export_test.go b/kadai3-2/en-ken/export_test.go index 691e9e2..f80150d 100644 --- a/kadai3-2/en-ken/export_test.go +++ b/kadai3-2/en-ken/export_test.go @@ -21,4 +21,8 @@ type TestRange struct { To int64 } -const MaxRangeSize = maxRangeSize +var MaxRangeSize = maxRangeSize + +func SetMaxRangeSize(size int64) { + maxRangeSize = size +} From 9dff956899b613456310bd2e2d46d976d21eb779 Mon Sep 17 00:00:00 2001 From: en-ken Date: Sun, 1 Sep 2019 13:15:48 +0900 Subject: [PATCH 15/15] doc: updated readme.md --- kadai3-2/en-ken/Readme.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/kadai3-2/en-ken/Readme.md b/kadai3-2/en-ken/Readme.md index 3a71f26..08eaa2c 100644 --- a/kadai3-2/en-ken/Readme.md +++ b/kadai3-2/en-ken/Readme.md @@ -8,7 +8,14 @@ - golang.org/x/sync/errgroupパッケージなどを使ってみる - キャンセルが発生した場合の実装を行う -## やること +## 使い方 + +``` +go get github.com/gopherdojo/dojo6/kadai3-2/en-ken/dl-mgr +div -n [goroutineの並列数(デフォルト:5)] -o [保存ファイル名(デフォルト:リモート名)] URL +``` + +## やったこと 1. HEADリクエストしてAccept-Rangesヘッダの有無確認 1. Accept-Rangesがあった場合、 @@ -18,5 +25,15 @@ ## 工夫した点 -- 1つのgoroutineがダウンロードするデータが1MBを超えた場合、1MBごとにファイル出力するようにした。 +- 1つのgoroutineがダウンロードするデータが1MBを超えた場合、1MBごとにファイル出力する。 +- Rangeアクセスに対応していなかったら普通にダウンロードする。 - すでにダウンロード済のデータがあった場合(そのパートのファイルがあった場合)、それを再利用する。 + +## 困っていること + +- `go test`のやり方がいまいちわかっていない。依存関係の解決の仕方が理解に至っていない。 + - `go test *.go`だと`export_test.go`が解決できない + - `go test ./`だと解決できる + - `go test ./...`すると`utils`以下がimport cycleで失敗してしまう + - `go test ./utils/*`だと`utils`だと問題ない + - `go test -cover ./utils/*`でちゃんとカバレッジがどれない