package utils import ( "compress/gzip" "context" "fmt" "io" "net/http" "os" ) var httpClient http.Client func DownloadE6Data(ctx context.Context, filename string, targetPath string) error { req, err := buildE6Request(fmt.Sprintf("/db_export/%s", filename)) if err != nil { return err } req = req.WithContext(ctx) resp, err := httpClient.Do(req) if err != nil { return err } defer resp.Body.Close() uncompressedStream, err := gzip.NewReader(resp.Body) if err != nil { return err } defer uncompressedStream.Close() out, err := os.Create(targetPath) if err != nil { return err } defer out.Close() _, err = io.Copy(out, uncompressedStream) return err } func buildE6Request(url string) (*http.Request, error) { request, err := http.NewRequest("GET", fmt.Sprintf("%s%s", "https://e621.net", url), nil) if err != nil { return nil, err } request.Header.Add("User-Agent", "Anthrove downloader (by alphyron)") return request, nil }