api, chunk, cmd, shed, storage: add support for pinning content (#1509)

* add support for pinning content as per the specifications defined initially in #1274 and as further discussed in https://swarmresear.ch/t/pinned-content-in-swarm

Author: jmozah

api/api.go

@@ -21,20 +21,19 @@ package api
 
 import (
 	"archive/tar"
+	"bytes"
 	"context"
 	"crypto/ecdsa"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
 	"math/big"
+	"mime"
 	"net/http"
 	"path"
-	"strings"
-
-	"bytes"
-	"mime"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -47,8 +46,7 @@ import (
 	"github.com/ethersphere/swarm/storage"
 	"github.com/ethersphere/swarm/storage/feed"
 	"github.com/ethersphere/swarm/storage/feed/lookup"
-
-	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go"
 )
 
 var (
@@ -400,6 +398,8 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 	return
 }
 
+// Delete handles removing a file from the manifest.
+// This creates a new manifest without the given path
 func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Address, error) {
 	apiDeleteCount.Inc(1)
 	uri, err := Parse("bzz:/" + addr)

api/client/client.go

@@ -63,7 +63,7 @@ type Client struct {
 
 // UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it
 // uploads encrypted data
-func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) {
+func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool, toPin bool) (string, error) {
 	if size <= 0 {
 		return "", errors.New("data size must be greater than zero")
 	}
@@ -78,6 +78,11 @@ func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, err
 	req.ContentLength = size
 	req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("raw_upload_%d", time.Now().Unix()))
 
+	// Set the pinning header if the file needs to be pinned
+	if toPin {
+		req.Header.Set(swarmhttp.PinHeaderName, "true")
+	}
+
 	res, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return "", err
@@ -151,11 +156,11 @@ func Open(path string) (*File, error) {
 // (if the manifest argument is non-empty) or creates a new manifest containing
 // the file, returning the resulting manifest hash (the file will then be
 // available at bzz:/<hash>/<path>)
-func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) {
+func (c *Client) Upload(file *File, manifest string, toEncrypt bool, toPin bool) (string, error) {
 	if file.Size <= 0 {
 		return "", errors.New("file size must be greater than zero")
 	}
-	return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt)
+	return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt, toPin)
 }
 
 // Download downloads a file with the given path from the swarm manifest with
@@ -185,7 +190,7 @@ func (c *Client) Download(hash, path string) (*File, error) {
 // directory will then be available at bzz:/<hash>/path/to/file), with
 // the file specified in defaultPath being uploaded to the root of the manifest
 // (i.e. bzz:/<hash>/)
-func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) {
+func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool, toPin bool) (string, error) {
 	stat, err := os.Stat(dir)
 	if err != nil {
 		return "", err
@@ -200,7 +205,7 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo
 			return "", fmt.Errorf("default path: %v", err)
 		}
 	}
-	return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt)
+	return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt, toPin)
 }
 
 // DownloadDirectory downloads the files contained in a swarm manifest under
@@ -358,12 +363,13 @@ func (c *Client) DownloadFile(hash, path, dest, credentials string) error {
 }
 
 // UploadManifest uploads the given manifest to swarm
-func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) {
+func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool, toPin bool) (string, error) {
 	data, err := json.Marshal(m)
 	if err != nil {
 		return "", err
 	}
-	return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
+
+	return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt, toPin)
 }
 
 // DownloadManifest downloads a swarm manifest
@@ -498,7 +504,7 @@ type UploadFn func(file *File) error
 
 // TarUpload uses the given Uploader to upload files to swarm as a tar stream,
 // returning the resulting manifest hash
-func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) {
+func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool, toPin bool) (string, error) {
 	ctx, sp := spancontext.StartSpan(context.Background(), "api.client.tarupload")
 	defer sp.Finish()
 
@@ -540,6 +546,11 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t
 
 	req.Header.Set(swarmhttp.SwarmTagHeaderName, tag)
 
+	// Set the pinning header if the file is to be pinned
+	if toPin {
+		req.Header.Set(swarmhttp.PinHeaderName, "true")
+	}
+
 	// use 'Expect: 100-continue' so we don't send the request body if
 	// the server refuses the request
 	req.Header.Set("Expect", "100-continue")
@@ -591,7 +602,7 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t
 
 // MultipartUpload uses the given Uploader to upload files to swarm as a
 // multipart form, returning the resulting manifest hash
-func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
+func (c *Client) MultipartUpload(hash string, uploader Uploader, toPin bool) (string, error) {
 	reqR, reqW := io.Pipe()
 	defer reqR.Close()
 	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
@@ -606,6 +617,9 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error)
 	mw := multipart.NewWriter(reqW)
 	req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
 	req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("multipart_upload_%d", time.Now().Unix()))
+	if toPin {
+		req.Header.Set(swarmhttp.PinHeaderName, "true")
+	}
 
 	// define an UploadFn which adds files to the multipart form
 	uploadFn := func(file *File) error {

api/client/client_test.go

@@ -36,39 +36,47 @@ import (
 )
 
 func serverFunc(api *api.API) swarmhttp.TestServer {
-	return swarmhttp.NewServer(api, "")
+	return swarmhttp.NewServer(api, nil, "")
 }
 
 // TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
 func TestClientUploadDownloadRaw(t *testing.T) {
-	testClientUploadDownloadRaw(false, t)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil, nil)
+	defer srv.Close()
+
+	data := []byte("foo123")
+	testClientUploadDownloadRaw(srv, false, t, data, false)
+
+	// check the tag was created successfully
+	tag := srv.Tags.All()[0]
+	testutil.CheckTag(t, tag, 1, 1, 0, 1)
 }
 
 func TestClientUploadDownloadRawEncrypted(t *testing.T) {
+
 	if testutil.RaceEnabled {
 		t.Skip("flaky with -race on Travis")
 		// See: https://github.com/ethersphere/go-ethereum/issues/1254
 	}
 
-	testClientUploadDownloadRaw(true, t)
-}
-
-func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
-	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil, nil)
 	defer srv.Close()
 
-	client := NewClient(srv.URL)
-
-	// upload some raw data
 	data := []byte("foo123")
-	hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
-	if err != nil {
-		t.Fatal(err)
-	}
+	testClientUploadDownloadRaw(srv, true, t, data, false)
 
 	// check the tag was created successfully
 	tag := srv.Tags.All()[0]
 	testutil.CheckTag(t, tag, 1, 1, 0, 1)
+}
+
+func testClientUploadDownloadRaw(srv *swarmhttp.TestSwarmServer, toEncrypt bool, t *testing.T, data []byte, toPin bool) string {
+	client := NewClient(srv.URL)
+
+	hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt, toPin)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// check we can download the same data
 	res, isEncrypted, err := client.DownloadRaw(hash)
@@ -86,6 +94,8 @@ func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
 	if !bytes.Equal(gotData, data) {
 		t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
 	}
+
+	return hash
 }
 
 // TestClientUploadDownloadFiles test uploading and downloading files to swarm
@@ -98,12 +108,13 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) {
 	testClientUploadDownloadFiles(true, t)
 }
 
-func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
-	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) string {
+
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil, nil)
 	defer srv.Close()
 
 	client := NewClient(srv.URL)
-	upload := func(manifest, path string, data []byte) string {
+	upload := func(manifest, path string, data []byte, toPin bool) string {
 		file := &File{
 			ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
 			ManifestEntry: api.ManifestEntry{
@@ -112,7 +123,7 @@ func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
 				Size:        int64(len(data)),
 			},
 		}
-		hash, err := client.Upload(file, manifest, toEncrypt)
+		hash, err := client.Upload(file, manifest, toEncrypt, toPin)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -141,25 +152,27 @@ func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
 
 	// upload a file to the root of a manifest
 	rootData := []byte("some-data")
-	rootHash := upload("", "", rootData)
+	rootHash := upload("", "", rootData, false)
 
 	// check we can download the root file
 	checkDownload(rootHash, "", rootData)
 
 	// upload another file to the same manifest
 	otherData := []byte("some-other-data")
-	newHash := upload(rootHash, "some/other/path", otherData)
+	newHash := upload(rootHash, "some/other/path", otherData, false)
 
 	// check we can download both files from the new manifest
 	checkDownload(newHash, "", rootData)
 	checkDownload(newHash, "some/other/path", otherData)
 
 	// replace the root file with different data
-	newHash = upload(newHash, "", otherData)
+	newHash = upload(newHash, "", otherData, false)
 
 	// check both files have the other data
 	checkDownload(newHash, "", otherData)
 	checkDownload(newHash, "some/other/path", otherData)
+
+	return newHash
 }
 
 var testDirFiles = []string{
@@ -194,10 +207,10 @@ func newTestDirectory(t *testing.T) string {
 	return dir
 }
 
-// TestClientUploadDownloadDirectory tests uploading and downloading a
+// TestClientUploadDownloadDirectory tests uploading and downloading
 // directory of files to a swarm manifest
 func TestClientUploadDownloadDirectory(t *testing.T) {
-	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil, nil)
 	defer srv.Close()
 
 	dir := newTestDirectory(t)
@@ -206,7 +219,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
 	// upload the directory
 	client := NewClient(srv.URL)
 	defaultPath := testDirFiles[0]
-	hash, err := client.UploadDirectory(dir, defaultPath, "", false)
+	hash, err := client.UploadDirectory(dir, defaultPath, "", false, false)
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)
 	}
@@ -267,14 +280,14 @@ func TestClientFileListEncrypted(t *testing.T) {
 }
 
 func testClientFileList(toEncrypt bool, t *testing.T) {
-	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil, nil)
 	defer srv.Close()
 
 	dir := newTestDirectory(t)
 	defer os.RemoveAll(dir)
 
 	client := NewClient(srv.URL)
-	hash, err := client.UploadDirectory(dir, "", "", toEncrypt)
+	hash, err := client.UploadDirectory(dir, "", "", toEncrypt, false)
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)
 	}
@@ -325,14 +338,14 @@ func testClientFileList(toEncrypt bool, t *testing.T) {
 // TestClientMultipartUpload tests uploading files to swarm using a multipart
 // upload
 func TestClientMultipartUpload(t *testing.T) {
-	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil, nil)
 	defer srv.Close()
 
 	// define an uploader which uploads testDirFiles with some data
 	// note: this test should result in SEEN chunks. assert accordingly
-	data := []byte("some-data")
 	uploader := UploaderFunc(func(upload UploadFn) error {
 		for _, name := range testDirFiles {
+			data := []byte(name)
 			file := &File{
 				ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
 				ManifestEntry: api.ManifestEntry{
@@ -350,14 +363,14 @@ func TestClientMultipartUpload(t *testing.T) {
 
 	// upload the files as a multipart upload
 	client := NewClient(srv.URL)
-	hash, err := client.MultipartUpload("", uploader)
+	hash, err := client.MultipartUpload("", uploader, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// check the tag was created successfully
 	tag := srv.Tags.All()[0]
-	testutil.CheckTag(t, tag, 9, 9, 7, 9)
+	testutil.CheckTag(t, tag, 9, 9, 0, 9)
 
 	// check we can download the individual files
 	checkDownloadFile := func(path string) {
@@ -370,8 +383,9 @@ func TestClientMultipartUpload(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if !bytes.Equal(gotData, data) {
-			t.Fatalf("expected data to be %q, got %q", data, gotData)
+		// The content is the file name just to make them different
+		if !bytes.Equal(gotData, []byte(path)) {
+			t.Fatalf("expected data to be %q, got %q", path, gotData)
 		}
 	}
 	for _, file := range testDirFiles {
@@ -398,7 +412,7 @@ func TestClientBzzWithFeed(t *testing.T) {
 	signer, _ := newTestSigner()
 
 	// Initialize a Swarm test server
-	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil, nil)
 	swarmClient := NewClient(srv.URL)
 	defer srv.Close()
 
@@ -433,7 +447,7 @@ func TestClientBzzWithFeed(t *testing.T) {
 	}
 
 	// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
-	manifestAddressHex, err := swarmClient.Upload(f, "", false)
+	manifestAddressHex, err := swarmClient.Upload(f, "", false, false)
 	if err != nil {
 		t.Fatalf("Error creating manifest: %s", err)
 	}
@@ -516,7 +530,7 @@ func TestClientCreateUpdateFeed(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
-	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil, nil)
 	client := NewClient(srv.URL)
 	defer srv.Close()