Merge pull request #44 from jdolitsky/azure-support

Support for Microsoft Azure Blob Storage

README.md

@@ -6,7 +6,7 @@
 [![GoDoc](https://godoc.org/github.com/kubernetes-helm/chartmuseum?status.svg)](https://godoc.org/github.com/kubernetes-helm/chartmuseum)
 <sub>**_"Preserve your precious artifacts... in the cloud!"_**<sub>
 
-*ChartMuseum* is an open-source **[Helm Chart Repository](https://github.com/kubernetes/helm/blob/master/docs/chart_repository.md)** written in Go (Golang), with support for cloud storage backends, including [Google Cloud Storage](https://cloud.google.com/storage/) and [Amazon S3](https://aws.amazon.com/s3/).
+*ChartMuseum* is an open-source **[Helm Chart Repository](https://github.com/kubernetes/helm/blob/master/docs/chart_repository.md)** written in Go (Golang), with support for cloud storage backends, including [Google Cloud Storage](https://cloud.google.com/storage/), [Amazon S3](https://aws.amazon.com/s3/), and [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/).
 
 Works as a valid Helm Chart Repository, and also provides an API for uploading new chart packages to storage etc.
 
@@ -171,6 +171,21 @@ chartmuseum --debug --port=8080 \
   --storage-google-prefix=""
 ```
 
+#### Using with Microsoft Azure Blob Storage
+
+Make sure your environment is properly setup to access `mycontainer`.
+
+To do so, you much set the following env vars:
+- `AZURE_STORAGE_ACCOUNT`
+- `AZURE_STORAGE_ACCESS_KEY`
+
+```bash
+chartmuseum --debug --port=8080 \
+  --storage="azure" \
+  --storage-microsoft-container="mycontainer" \
+  --storage-microsoft-prefix=""
+```
+
 #### Using with local filesystem storage
 Make sure you have read-write access to `./chartstorage` (will create if doesn't exist)
 ```bash

acceptance_tests/helm.robot

@@ -19,14 +19,20 @@ ChartMuseum works with Helm using Amazon cloud storage
 ChartMuseum works with Helm using Google cloud storage
     Test Helm integration   google
 
+ChartMuseum works with Helm using Microsoft cloud storage
+    Test Helm integration   microsoft
+
 *** Keyword ***
 Test Helm integration
     [Arguments]    ${storage}
 
-    # return fast if we cannot find a bucket in an environment variable.
+    # return fast if we cannot find a bucket/container in an environment variable.
     ${USTORAGE}=  Convert To Uppercase  ${storage}
-    ${ENV_STORAGE_SET}=  Get Environment variable  TEST_STORAGE_${USTORAGE}_BUCKET  ${EMPTY}
-    Return from Keyword if  '${ENV_STORAGE_SET}'=='${EMPTY}' and '${storage}'!='local'
+    ${ENV_STORAGE_BUCKET_SET}=  Get Environment variable  TEST_STORAGE_${USTORAGE}_BUCKET  ${EMPTY}
+    Return from Keyword if  '${ENV_STORAGE_BUCKET_SET}'=='${EMPTY}' and '${storage}'!='local' and '${storage}'!='microsoft'
+    ${ENV_STORAGE_CONTAINER_SET}=  Get Environment variable  TEST_STORAGE_${USTORAGE}_CONTAINER  ${EMPTY}
+    Return from Keyword if  '${ENV_STORAGE_CONTAINER_SET}'=='${EMPTY}' and '${storage}'=='microsoft'
+    ${ENV_STORAGE_CONTAINER_SET}=  Get Environment variable  TEST_STORAGE_${USTORAGE}_CONTAINER  ${EMPTY}
 
     Start ChartMuseum server with storage backend  ${storage}
     Able to add ChartMuseum as Helm chart repo

acceptance_tests/lib/ChartMuseum.py

@@ -25,6 +25,9 @@ class ChartMuseum(common.CommandRunner):
         elif storage == 'google':
             cmd += '--storage-google-bucket="%s" --storage-google-prefix="%s" >> %s 2>&1' \
                    % (common.STORAGE_GOOGLE_BUCKET, common.STORAGE_GOOGLE_PREFIX, common.LOGFILE)
+        elif storage == 'microsoft':
+            cmd += '--storage-microsoft-container="%s" --storage-microsoft-prefix="%s"  >> %s 2>&1' \
+                   % (common.STORAGE_MICROSOFT_CONTAINER, common.STORAGE_MICROSOFT_PREFIX, common.LOGFILE)
         print(cmd)
         self.run_command(cmd, detach=True)
 

acceptance_tests/lib/common.py

@@ -15,9 +15,11 @@ LOGFILE = '.chartmuseum.log'
 STORAGE_AMAZON_BUCKET = os.getenv('TEST_STORAGE_AMAZON_BUCKET')
 STORAGE_AMAZON_REGION = os.getenv('TEST_STORAGE_AMAZON_REGION')
 STORAGE_GOOGLE_BUCKET = os.getenv('TEST_STORAGE_GOOGLE_BUCKET')
+STORAGE_MICROSOFT_CONTAINER = os.getenv('TEST_STORAGE_MICROSOFT_CONTAINER')
 
 STORAGE_AMAZON_PREFIX = 'acceptance/%s' % NOW
 STORAGE_GOOGLE_PREFIX = 'acceptance/%s' % NOW
+STORAGE_MICROSOFT_PREFIX = 'acceptance/%s' % NOW
 
 
 class CommandRunner(object):

cmd/chartmuseum/main.go

@@ -82,6 +82,8 @@ func backendFromContext(c *cli.Context) storage.Backend {
 		backend = amazonBackendFromContext(c)
 	case "google":
 		backend = googleBackendFromContext(c)
+	case "microsoft":
+		backend = microsoftBackendFromContext(c)
 	default:
 		crash("Unsupported storage backend: ", storageFlag)
 	}
@@ -119,6 +121,14 @@ func googleBackendFromContext(c *cli.Context) storage.Backend {
 	))
 }
 
+func microsoftBackendFromContext(c *cli.Context) storage.Backend {
+	crashIfContextMissingFlags(c, []string{"storage-microsoft-container"})
+	return storage.Backend(storage.NewMicrosoftBlobBackend(
+		c.String("storage-microsoft-container"),
+		c.String("storage-microsoft-prefix"),
+	))
+}
+
 func crashIfContextMissingFlags(c *cli.Context, flags []string) {
 	missing := []string{}
 	for _, flag := range flags {
@@ -243,6 +253,16 @@ var cliFlags = []cli.Flag{
 		Usage:  "prefix to store charts for --storage-google-bucket",
 		EnvVar: "STORAGE_GOOGLE_PREFIX",
 	},
+	cli.StringFlag{
+		Name:   "storage-microsoft-container",
+		Usage:  "container to store charts for microsoft storage backend",
+		EnvVar: "STORAGE_MICROSOFT_CONTAINER",
+	},
+	cli.StringFlag{
+		Name:   "storage-microsoft-prefix",
+		Usage:  "prefix to store charts for --storage-microsoft-prefix",
+		EnvVar: "STORAGE_MICROSOFT_PREFIX",
+	},
 	cli.StringFlag{
 		Name:   "chart-post-form-field-name",
 		Value:  "chart",

glide.lock

-hash: 71e41ce9ed31cd3679a3f0674c55379782959ec21a40e6138a88db301cf8a75e
-updated: 2018-02-08T14:25:41.542372301-08:00
+hash: 672d101b43f4de4dd3122ca9e492d31db6b0087cf3cabbda7ab545134d2aa3bc
+updated: 2018-02-13T11:03:26.68753-06:00
 imports:
 - name: cloud.google.com/go
   version: 3137f1def9552929c7beb11f2ac7d2dd998e4040
@@ -44,12 +44,26 @@ imports:
   - service/s3/s3iface
   - service/s3/s3manager
   - service/sts
+- name: github.com/Azure/azure-sdk-for-go
+  version: 5bf28521f7a6f72f2dde3e5a5fb0d27d7a8aeee9
+  subpackages:
+  - storage
+  - version
+- name: github.com/Azure/go-autorest
+  version: c67b24a8e30d876542a85022ebbdecf0e5a935e8
+  subpackages:
+  - autorest
+  - autorest/adal
+  - autorest/azure
+  - autorest/date
 - name: github.com/beorn7/perks
   version: 3ac7bf7a47d159a033b107610db8a1b6575507a4
   subpackages:
   - quantile
 - name: github.com/BurntSushi/toml
   version: b26d9c308763d68093482582cea63d69be07a0f0
+- name: github.com/dgrijalva/jwt-go
+  version: dbeaa9332f19a944acb5736b4456cfcc02140e29
 - name: github.com/ghodss/yaml
   version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee
 - name: github.com/gin-contrib/sse
@@ -84,6 +98,8 @@ imports:
   version: 2cadd475a3e966ec9b77a21afc530dbacec6d613
 - name: github.com/jmespath/go-jmespath
   version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
+- name: github.com/marstr/guid
+  version: 8bdf7d1a087ccc975cf37dd6507da50698fd19ca
 - name: github.com/Masterminds/semver
   version: 517734cc7d6470c0d07130e40fd40bdeb9bcd3fd
 - name: github.com/mattn/go-isatty

glide.yaml

@@ -16,6 +16,12 @@ import:
 - package: github.com/atarantini/ginrequestid
   version: a432ade0ce478903613da1372cb5a62914cfe532
 
+# Used by Microsoft Azure Blob storage backend
+- package: github.com/Azure/azure-sdk-for-go
+  version: v14.0.0
+  subpackages:
+  - storage
+
 # these ones are srsly a pain in da butt...
 # all needed to get cloud.google.com/go/storage to work
 - package: github.com/golang/protobuf

pkg/storage/microsoft.go

+package storage
+
+import (
+	"errors"
+	"io/ioutil"
+	"time"
+	pathutil "path"
+
+	microsoft_storage "github.com/Azure/azure-sdk-for-go/storage"
+	"os"
+)
+
+// MicrosoftBlobBackend is a storage backend for Microsoft Azure Blob Storage
+type MicrosoftBlobBackend struct {
+	Prefix    string
+	Container *microsoft_storage.Container
+}
+
+// NewMicrosoftBlobBackend creates a new instance of MicrosoftBlobBackend
+func NewMicrosoftBlobBackend(container string, prefix string) *MicrosoftBlobBackend {
+
+	// From the Azure portal, get your storage account name and key and set environment variables.
+	accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if len(accountName) == 0 || len(accountKey) == 0 {
+		panic("Either the AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY environment variable is not set")
+	}
+
+	client, err := microsoft_storage.NewBasicClient(accountName, accountKey)
+	if err != nil {
+		panic(err)
+	}
+
+	blobClient := client.GetBlobService()
+	containerRef := blobClient.GetContainerReference(container)
+
+	b := &MicrosoftBlobBackend{
+		Prefix:    prefix,
+		Container: containerRef,
+	}
+
+	return b
+}
+
+// ListObjects lists all objects in Microsoft Azure Blob Storage container
+func (b MicrosoftBlobBackend) ListObjects() ([]Object, error) {
+	var objects []Object
+
+	if b.Container == nil {
+		return objects, errors.New("Unable to obtain a container reference.")
+	}
+
+	var params microsoft_storage.ListBlobsParameters
+	params.Prefix = b.Prefix
+	response, err := b.Container.ListBlobs(params)
+	if err != nil {
+		return objects, err
+	}
+
+	for _, blob := range response.Blobs {
+		path := removePrefixFromObjectPath(b.Prefix, blob.Name)
+		if objectPathIsInvalid(path) {
+			continue
+		}
+
+		err = blob.GetProperties(nil)
+		if err != nil {
+			return objects, err
+		}
+
+		object := Object{
+			Path:         path,
+			Content:      []byte{},
+			LastModified: time.Time(blob.Properties.LastModified),
+		}
+
+		objects = append(objects, object)
+	}
+	return objects, nil
+}
+
+// GetObject retrieves an object from Microsoft Azure Blob Storage, at path
+func (b MicrosoftBlobBackend) GetObject(path string) (Object, error) {
+	var object Object
+	object.Path = path
+
+	if b.Container == nil {
+		return object, errors.New("Unable to obtain a container reference.")
+	}
+
+	var content []byte
+
+	blobReference := b.Container.GetBlobReference(pathutil.Join(b.Prefix, path))
+	exists, err := blobReference.Exists()
+	if err != nil {
+		return object, err
+	}
+
+	if !exists {
+		return object, errors.New("Object does not exist.")
+	}
+
+	readCloser, err := blobReference.Get(nil)
+	if err != nil {
+		return object, err
+	}
+
+	content, err = ioutil.ReadAll(readCloser)
+	if err != nil {
+		return object, err
+	}
+
+	object.Content = content
+	err = blobReference.GetProperties(nil)
+	object.LastModified = time.Time(blobReference.Properties.LastModified)
+	return object, nil
+}
+
+// PutObject uploads an object to Microsoft Azure Blob Storage container, at path
+func (b MicrosoftBlobBackend) PutObject(path string, content []byte) error {
+	if b.Container == nil {
+		return errors.New("Unable to obtain a container reference.")
+	}
+
+	blobReference := b.Container.GetBlobReference(pathutil.Join(b.Prefix, path))
+
+	err := blobReference.PutAppendBlob(nil)
+	if err == nil {
+		err = blobReference.AppendBlock(content, nil)
+	}
+
+	return err
+}
+
+// DeleteObject removes an object from Microsoft Azure Blob Storage container, at path
+func (b MicrosoftBlobBackend) DeleteObject(path string) error {
+	if b.Container == nil {
+		return errors.New("Unable to obtain a container reference.")
+	}
+
+	blobReference := b.Container.GetBlobReference(pathutil.Join(b.Prefix, path))
+	_, err := blobReference.DeleteIfExists(nil)
+	return err
+}

pkg/storage/microsoft_test.go

+package storage
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type MicrosoftTestSuite struct {
+	suite.Suite
+	BrokenAzureBlobBackend   *MicrosoftBlobBackend
+	NoPrefixAzureBlobBackend *MicrosoftBlobBackend
+}
+
+func (suite *MicrosoftTestSuite) SetupSuite() {
+	backend := NewMicrosoftBlobBackend("fake-container-cant-exist-fbce123", "")
+	suite.BrokenAzureBlobBackend = backend
+
+	containerName := os.Getenv("TEST_STORAGE_MICROSOFT_CONTAINER")
+	backend = NewMicrosoftBlobBackend(containerName, "")
+	suite.NoPrefixAzureBlobBackend = backend
+
+	data := []byte("some object")
+	path := "deleteme.txt"
+	err := suite.NoPrefixAzureBlobBackend.PutObject(path, data)
+	suite.Nil(err, "no error putting deleteme.txt using Azure backend")
+}
+
+func (suite *MicrosoftTestSuite) TearDownSuite() {
+	err := suite.NoPrefixAzureBlobBackend.DeleteObject("deleteme.txt")
+	suite.Nil(err, "no error deleting deleteme.txt using Azure backend")
+}
+
+func (suite *MicrosoftTestSuite) TestListObjects() {
+	_, err := suite.BrokenAzureBlobBackend.ListObjects()
+	suite.NotNil(err, "cannot list objects with bad bucket")
+
+	_, err = suite.NoPrefixAzureBlobBackend.ListObjects()
+	suite.Nil(err, "can list objects with good bucket, no prefix")
+}
+
+func (suite *MicrosoftTestSuite) TestGetObject() {
+	_, err := suite.BrokenAzureBlobBackend.GetObject("this-file-cannot-possibly-exist.tgz")
+	suite.NotNil(err, "cannot get objects with bad bucket")
+}
+
+func (suite *MicrosoftTestSuite) TestPutObject() {
+	err := suite.BrokenAzureBlobBackend.PutObject("this-file-will-not-upload.txt", []byte{})
+	suite.NotNil(err, "cannot put objects with bad bucket")
+}
+
+func TestAzureStorageTestSuite(t *testing.T) {
+	if os.Getenv("TEST_CLOUD_STORAGE") == "1" &&
+		os.Getenv("TEST_STORAGE_AZURE_CONTAINER") != "" {
+		suite.Run(t, new(MicrosoftTestSuite))
+	}
+}

pkg/storage/storage_test.go

@@ -30,12 +30,16 @@ func (suite *StorageTestSuite) setupStorageBackends() {
 		s3Bucket := os.Getenv("TEST_STORAGE_AMAZON_BUCKET")
 		s3Region := os.Getenv("TEST_STORAGE_AMAZON_REGION")
 		gcsBucket := os.Getenv("TEST_STORAGE_GOOGLE_BUCKET")
+		blobContainer := os.Getenv("TEST_STORAGE_MICROSOFT_CONTAINER")
 		if s3Bucket != "" && s3Region != "" {
 			suite.StorageBackends["AmazonS3"] = Backend(NewAmazonS3Backend(s3Bucket, prefix, s3Region, "", ""))
 		}
 		if gcsBucket != "" {
 			suite.StorageBackends["GoogleCS"] = Backend(NewGoogleCSBackend(gcsBucket, prefix))
 		}
+		if blobContainer != "" {
+			suite.StorageBackends["MicrosoftBlob"] = Backend(NewMicrosoftBlobBackend(blobContainer, prefix))
+		}
 	}
 }
 

scripts/setup_test_environment.sh

@@ -5,6 +5,7 @@ REQUIRED_TEST_STORAGE_ENV_VARS=(
     "TEST_STORAGE_AMAZON_BUCKET"
     "TEST_STORAGE_AMAZON_REGION"
     "TEST_STORAGE_GOOGLE_BUCKET"
+    "TEST_STORAGE_MICROSOFT_CONTAINER"
 )
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"