cpod

  1// Copyright (C) 2013-2015 Sören Tempel
  2//
  3// This program is free software: you can redistribute it and/or modify
  4// it under the terms of the GNU General Public License as published by
  5// the Free Software Foundation, either version 3 of the License, or
  6// (at your option) any later version.
  7//
  8// This program is distributed in the hope that it will be useful,
  9// but WITHOUT ANY WARRANTY; without even the implied warranty of
 10// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 11// GNU General Public License for more details.
 12//
 13// You should have received a copy of the GNU General Public License
 14// along with this program. If not, see <http://www.gnu.org/licenses/>.
 15
 16package util
 17
 18import (
 19	"errors"
 20	"fmt"
 21	"io"
 22	"net"
 23	"net/http"
 24	"net/url"
 25	"os"
 26	"path"
 27	"path/filepath"
 28	"strings"
 29	"time"
 30)
 31
 32const (
 33	// Number of times a failed HTTP request is retried.
 34	retry = 3
 35
 36	// Number of maximal allowed redirects.
 37	maxRedirects = 10
 38
 39	// HTTP User-Agent.
 40	useragent = "cpod"
 41)
 42
 43// Get performs a HTTP GET request, just like http.get, however, it has
 44// a few handy extra features: I adds a User-Agent header and it retries
 45// a failed get request if the error was a temporary one.
 46func Get(uri string) (resp *http.Response, err error) {
 47	req, err := http.NewRequest("GET", uri, nil)
 48	if err != nil {
 49		return
 50	}
 51
 52	return doReq(req)
 53}
 54
 55// GetFile downloads the file from the given uri and stores it in the
 56// specified target directory. If a download was interrupted previously
 57// GetFile is able to resume it.
 58func GetFile(uri, target string) (fp string, err error) {
 59	if err = os.MkdirAll(target, 0755); err != nil {
 60		return
 61	}
 62
 63	fn, err := filename(uri)
 64	if err != nil {
 65		return
 66	}
 67
 68	fp = filepath.Join(target, fn)
 69	partPath := fmt.Sprintf("%s.part", fp)
 70	if _, err = os.Open(partPath); os.IsNotExist(err) {
 71		if err = newGet(uri, partPath); err != nil {
 72			return
 73		}
 74	} else {
 75		if err = resumeGet(uri, partPath); err != nil {
 76			return
 77		}
 78	}
 79
 80	if err = os.Rename(partPath, fp); err != nil {
 81		return
 82	}
 83
 84	return
 85}
 86
 87// resumeGet resumes an canceled download started by the newGet
 88// function.
 89func resumeGet(uri, target string) error {
 90	fi, err := os.Stat(target)
 91	if err != nil {
 92		return err
 93	}
 94
 95	req, err := http.NewRequest("GET", uri, nil)
 96	if err != nil {
 97		return err
 98	}
 99
100	req.Header.Add("Range", fmt.Sprintf("bytes=%d-", fi.Size()))
101	req.Header.Add("If-Range", fi.ModTime().Format(time.RFC1123))
102
103	resp, err := doReq(req)
104	if err != nil {
105		return err
106	}
107
108	defer resp.Body.Close()
109	if resp.StatusCode != http.StatusPartialContent {
110		return newGet(uri, target)
111	}
112
113	file, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0644)
114	if err != nil {
115		return err
116	}
117
118	defer file.Close()
119	if _, err = io.Copy(file, resp.Body); err != nil {
120		return err
121	}
122
123	return nil
124}
125
126// newGet starts a new file download, if the download wasn't completed
127// it can be resumed later on using the resumeGet function.
128func newGet(uri, target string) error {
129	resp, err := Get(uri)
130	if err != nil {
131		return err
132	}
133
134	reader := resp.Body
135	defer reader.Close()
136
137	file, err := os.Create(target)
138	if err != nil {
139		return err
140	}
141
142	defer file.Close()
143	if _, err = io.Copy(file, reader); err != nil {
144		return err
145	}
146
147	return err
148}
149
150// filename returns the fiilename of an URL. Basically it just uses
151// path.Base to determine the filename but it also removes queries.
152// Furthermore it also guarantees that the filename is not empty by
153// setting it to "unnamed" if it couldn't determine a proper filename.
154func filename(uri string) (fn string, err error) {
155	u, err := url.Parse(uri)
156	if err != nil {
157		return
158	}
159
160	fn = strings.TrimSpace(path.Base(u.Path))
161	if len(fn) <= 0 || fn == "/" || fn == "." {
162		fn = "unnamed"
163	}
164
165	return
166}
167
168// doReq does the same as net.client.Do but it retries sending the request if a
169// temporary error on layer 4 is encountered. Furthermore, it also ensure that
170// headers remain the same after a redirect and it adds a User-Agent header.
171func doReq(req *http.Request) (resp *http.Response, err error) {
172	req.Header.Add("User-Agent", useragent)
173	client := headerClient(req.Header)
174
175	for i := 1; i <= retry; i++ {
176		resp, err = client.Do(req)
177		if nerr, ok := err.(net.Error); ok && (nerr.Temporary() || nerr.Timeout()) {
178			time.Sleep(time.Duration(i*3) * time.Second)
179		} else {
180			break
181		}
182	}
183
184	return
185}
186
187// headerClient returns a client witch a custom CheckRedirect function
188// which ensures that the given headers will be readded after a redirect.
189func headerClient(headers http.Header) *http.Client {
190	redirectFunc := func(req *http.Request, via []*http.Request) error {
191		if len(via) >= maxRedirects {
192			return errors.New("too many redirects")
193		}
194
195		req.Header = headers
196		return nil
197	}
198
199	return &http.Client{CheckRedirect: redirectFunc}
200}