Added support for function deployment from gzip/bzip2/xz compressed tar files

.circleci/config.yml

@@ -27,6 +27,7 @@ exports: &exports
     echo "export FUNCTION_IMAGE_BUILDER=${BUILDER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV
     echo "export KUBECFG_JPATH=/home/circleci/src/github.com/kubeless/kubeless/ksonnet-lib" >> $BASH_ENV
     echo "export PATH=$(pwd)/bats/libexec:$GOPATH/bin:$PATH" >> $BASH_ENV
+    echo "export GIT_SHA1=${CIRCLE_SHA1}" >> $BASH_ENV
 restore_workspace: &restore_workspace
   run: |
     make bootstrap
@@ -112,6 +113,8 @@ jobs:
       image: ubuntu-1604:201903-01
     steps:
       - checkout
+      - run: sudo apt-get update -y
+      - run: sudo apt-get install -y tar gzip bzip2 xz-utils
       - attach_workspace:
           at: /tmp/go
       - <<: *exports

cmd/kubeless/function/function_test.go

@@ -17,7 +17,11 @@ limitations under the License.
 package function
 
 import (
+	"archive/tar"
 	"archive/zip"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -331,42 +335,81 @@ func TestGetFunctionDescription(t *testing.T) {
 		t.Errorf("Unexpected result. Expecting:\n %+v\n Received %+v\n", newFunction, *result3)
 	}
 
-	// It should detect that it is a Zip file
+	// It should detect that it is a Zip file or a compressed tar file
 	file, err = os.Open(file.Name())
 	if err != nil {
 		t.Error(err)
 	}
-	newfile, err := os.Create(file.Name() + ".zip")
+
+	zipFile, err := os.Create(file.Name() + ".zip")
+	if err != nil {
+		t.Error(err)
+	}
+	defer os.Remove(zipFile.Name()) // clean up
+
+	tarGzFile, err := os.Create(file.Name() + ".tar.gz")
 	if err != nil {
 		t.Error(err)
 	}
-	defer os.Remove(newfile.Name()) // clean up
-	zipW := zip.NewWriter(newfile)
+	defer os.Remove(tarGzFile.Name()) // clean up
+
+	zipW := zip.NewWriter(zipFile)
+	gzipW := gzip.NewWriter(tarGzFile)
+	tarW := tar.NewWriter(gzipW)
+
 	info, err := file.Stat()
 	if err != nil {
 		t.Error(err)
 	}
-	header, err := zip.FileInfoHeader(info)
+
+	zipHeader, err := zip.FileInfoHeader(info)
+	if err != nil {
+		t.Error(err)
+	}
+	writer, err := zipW.CreateHeader(zipHeader)
+	if err != nil {
+		t.Error(err)
+	}
+	_, err = io.Copy(writer, file)
+	if err != nil {
+		t.Error(err)
+	}
+
+	tarHeader, err := tar.FileInfoHeader(info, info.Name())
 	if err != nil {
 		t.Error(err)
 	}
-	writer, err := zipW.CreateHeader(header)
+	tarHeader.Name = file.Name()
+	err = tarW.WriteHeader(tarHeader)
 	if err != nil {
 		t.Error(err)
 	}
 	_, err = io.Copy(writer, file)
 	if err != nil {
 		t.Error(err)
 	}
+
 	file.Close()
 	zipW.Close()
+	zipFile.Close()
+	tarW.Close()
+	gzipW.Close()
+	tarGzFile.Close()
 
-	result4, err := getFunctionDescription("test", "default", "file.handler", newfile.Name(), "dependencies", "runtime", "", "", "", "", "Always", "", 8080, 0, false, []string{}, []string{}, []string{}, []string{}, expectedFunction)
+	result4A, err := getFunctionDescription("test", "default", "file.handler", zipFile.Name(), "dependencies", "runtime", "", "", "", "", "Always", "", 8080, 0, false, []string{}, []string{}, []string{}, []string{}, expectedFunction)
 	if err != nil {
 		t.Error(err)
 	}
-	if result4.Spec.FunctionContentType != "base64+zip" {
-		t.Errorf("Should return base64+zip, received %s", result4.Spec.FunctionContentType)
+	if result4A.Spec.FunctionContentType != "base64+zip" {
+		t.Errorf("Should return base64+zip, received %s", result4A.Spec.FunctionContentType)
+	}
+
+	result4B, err := getFunctionDescription("test", "default", "file.handler", tarGzFile.Name(), "dependencies", "runtime", "", "", "", "", "Always", "", 8080, 0, false, []string{}, []string{}, []string{}, []string{}, expectedFunction)
+	if err != nil {
+		t.Error(err)
+	}
+	if result4B.Spec.FunctionContentType != "base64+compressedtar" {
+		t.Errorf("Should return base64+compressedtar, received %s", result4B.Spec.FunctionContentType)
 	}
 
 	// It should maintain previous HPA definition
@@ -496,30 +539,65 @@ func TestGetFunctionDescription(t *testing.T) {
 	if !reflect.DeepEqual(expectedURLFunction, *result7) {
 		t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result7)
 	}
-	// end test
 
-	// it should handle zip files from a URL and detect url+zip encoding
-	zipBytes, err := ioutil.ReadFile(newfile.Name())
+	// It should handle zip files and compressed tar files from a URL and detect url+zip and url+compressedtar encoding respectively
+	zipBytes, err := ioutil.ReadFile(zipFile.Name())
 	if err != nil {
 		t.Error(err)
 	}
 
-	ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts2A := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Write(zipBytes)
 	}))
-	defer ts2.Close()
+	defer ts2A.Close()
 
 	expectedURLFunction.Spec.FunctionContentType = "url+zip"
-	expectedURLFunction.Spec.Function = ts2.URL + "/test.zip"
-	result8, err := getFunctionDescription("test", "default", "file.handler", ts2.URL+"/test.zip", "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", "serviceAccount", 8080, 0, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, []string{}, kubelessApi.Function{})
+	expectedURLFunction.Spec.Function = ts2A.URL + "/test.zip"
+	expectedURLFunction.Spec.Checksum, err = getSha256(zipBytes)
+	if err != nil {
+		t.Error(err)
+	}
+
+	result8A, err := getFunctionDescription("test", "default", "file.handler", ts2A.URL+"/test.zip", "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", "serviceAccount", 8080, 0, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, []string{"foo3=bar3", "baz3:qux3"}, kubelessApi.Function{})
 	if err != nil {
 		t.Error(err)
 	}
-	if result8.Spec.FunctionContentType != "url+zip" {
-		t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8)
+	if !reflect.DeepEqual(expectedURLFunction, *result8A) {
+		t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8A)
 	}
-	if result8.Spec.Function != ts2.URL+"/test.zip" {
-		t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8)
+
+	tarGzBytes, err := ioutil.ReadFile(tarGzFile.Name())
+	if err != nil {
+		t.Error(err)
+	}
+	ts2B := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(tarGzBytes)
+	}))
+	defer ts2B.Close()
+
+	expectedURLFunction.Spec.FunctionContentType = "url+compressedtar"
+	expectedURLFunction.Spec.Function = ts2B.URL + "/test.tar.gz"
+	expectedURLFunction.Spec.Checksum, err = getSha256(tarGzBytes)
+	if err != nil {
+		t.Error(err)
+	}
+
+	result8B, err := getFunctionDescription("test", "default", "file.handler", ts2B.URL+"/test.tar.gz", "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", "serviceAccount", 8080, 0, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, []string{"foo3=bar3", "baz3:qux3"}, kubelessApi.Function{})
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(expectedURLFunction, *result8B) {
+		t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8B)
 	}
 	// end test
 }
+
+func getSha256(bytes []byte) (string, error) {
+	h := sha256.New()
+	_, err := h.Write(bytes)
+	if err != nil {
+		return "", err
+	}
+	checksum := hex.EncodeToString(h.Sum(nil))
+	return "sha256:" + checksum, nil
+}

docker/unzip/Dockerfile

@@ -1,2 +1,2 @@
 FROM bitnami/minideb
-RUN install_packages unzip curl ca-certificates
+RUN install_packages unzip curl ca-certificates tar gzip bzip2 xz-utils

docs/advanced-function-deployment.md

@@ -27,10 +27,10 @@ The fields that a Function specification can contain are:
 
  - Runtime: Runtime ID and version that the function will use. It should match one of the availables in the [Kubeless configuration](/docs/function-controller-configuration).
  - Timeout: Maximum timeout for the given function. After that time, the function execution will be terminated.
- - Handler: Pair of `<file_name>.<function_name>`. When using `zip` in `function-content-type` the `<file_name>` will be used to find the file with the function to expose. In other case it will be used just as a final file name. `<function_name>` is used to select the function to run from the exported functions of `<file_name>`. This field is mandatory and should match with an exported function.
+ - Handler: Pair of `<file_name>.<function_name>`. When using `zip` or `compressedtar` in `function-content-type`, the `<file_name>` will be used to find the file with the function to expose. In other cases, it will be used just as a final file name. `<function_name>` is used to select the function to run from the exported functions of `<file_name>`. This field is mandatory and should match with an exported function.
  - Deps: Dependencies of the function. The format of this field will depend on the runtime, e.g. a `package.json` for NodeJS functions or a `Gemfile` for Ruby.
  - Checksum: SHA256 of the function content.
- - Function content type: Content type of the function. Current supported values are `base64`, `url` or `text`. If the content is zipped the suffix `+zip` should be added.
+ - Function content type: Content type of the function. Current supported values are `base64`, `url` or `text`. If the content is zipped, the suffix `+zip` should be added. If the content is a gzip/bzip2/xz compressed tar file, the suffix `+compressedtar` should be added.
  - Function: Function content.
 
 Apart from the basic parameters, it is possible to add the specification of a `Deployment`, a `Service` or an `Horizontal Pod Autoscaler` that Kubeless will use to generate them.

docs/function-controller-configuration.md

@@ -235,7 +235,7 @@ It is possible to configure the different images that Kubeless uses for deploy a
   - Runtime Image: Image used to execute the function.
   - Init Image: Image used for installing the function and/or dependencies.
   - (Optional) Image Pull Secrets: Secret required to pull the image in case the repository is private.
- - The image used to populate the base image with the function. This is called `provision-image`. This image should have at least `unzip` and `curl`. It is also possible to specify `provision-image-secret` to specify a secret to pull that image from a private registry. 
+ - The image used to populate the base image with the function. This is called `provision-image`. This image should have at least `unzip`, `GNU tar`, `gzip`, `bzip2`, `xz` and `curl`. It is also possible to specify `provision-image-secret` to specify a secret to pull that image from a private registry.
  - The image used to build function images. This is called `builder-image`. This image is optional since its usage can be disabled with the property `enable-build-step`. A Dockerfile to build this image can be found [here](https://github.com/kubeless/kubeless/tree/master/docker/function-image-builder). It is also possible to specify `builder-image-secret` to specify a secret to pull that image from a private registry.
 
 ## Authenticate Kubeless Function Controller using OAuth Bearer Token

docs/quick-start.md

@@ -89,7 +89,7 @@ Let's dissect the command:
 
 * `hello`: This is the name of the function we want to deploy.
 * `--runtime python2.7`: This is the runtime we want to use to run our function. Available runtimes can be found executing `kubeless get-server-config`.
-* `--from-file test.py`: This is the file containing the function code. It is supported to specify a zip file as far as it doesn't exceed the maximum size for an etcd entry (1 MB).
+* `--from-file test.py`: This is the file containing the function code. Specifying a zip file or a gzip/bzip2/xz compressed tar file (see [list of supported suffixes](https://en.wikipedia.org/wiki/Tar_(computing)#Suffixes_for_compressed_files) for compressed tar files) is supported as long as it doesn't exceed the maximum size for an etcd entry (1 MB).
 * `--handler test.hello`: This specifies the file and the exposed function that will be used when receiving requests. In this example we are using the function `hello` from the file `test.py`.
 
 You can find the rest of options available when deploying a function executing `kubeless function deploy --help`

examples/Makefile

@@ -1,3 +1,6 @@
+GIT_SHA1 ?= master
+BASE_URL := https://raw.githubusercontent.com/kubeless/kubeless/$(GIT_SHA1)
+
 get-python:
 	kubeless function deploy get-python --runtime python2.7 --handler helloget.foo --from-file python/helloget.py
 
@@ -15,12 +18,33 @@ get-python-update-verify:
 	kubeless function call get-python |egrep hello.world.updated
 
 get-python-deps:
-	cd python && zip hellowithdeps.zip hellowithdeps.py  hellowithdepshelper.py && cd ..
+	cd python && zip hellowithdeps.zip hellowithdeps.py hellowithdepshelper.py && cd ..
 	kubeless function deploy get-python-deps --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.zip --dependencies python/requirements.txt
 
+get-python-deps-tar-gz:
+	cd python && tar czf hellowithdeps.tar.gz hellowithdeps.py hellowithdepshelper.py && cd ..
+	kubeless function deploy get-python-deps-tar-gz --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.tar.gz --dependencies python/requirements.txt
+
+get-python-deps-tar-bz2:
+	cd python && tar cjf hellowithdeps.tar.bz2 hellowithdeps.py hellowithdepshelper.py && cd ..
+	kubeless function deploy get-python-deps-tar-bz2 --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.tar.bz2 --dependencies python/requirements.txt
+
+get-python-deps-tar-xz:
+	cd python && tar cJf hellowithdeps.tar.xz hellowithdeps.py hellowithdepshelper.py && cd ..
+	kubeless function deploy get-python-deps-tar-xz --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.tar.xz --dependencies python/requirements.txt
+
 get-python-deps-verify:
 	kubeless function call get-python-deps |egrep Google
 
+get-python-deps-tar-gz-verify:
+	kubeless function call get-python-deps-tar-gz |egrep Google
+
+get-python-deps-tar-bz2-verify:
+	kubeless function call get-python-deps-tar-bz2 |egrep Google
+
+get-python-deps-tar-xz-verify:
+	kubeless function call get-python-deps-tar-xz |egrep Google
+
 get-python-custom-port:
 	kubeless function deploy get-python-custom-port --runtime python2.7 --handler helloget.foo --from-file python/helloget.py --port 8081
 
@@ -52,17 +76,36 @@ get-python-36-verify:
 	kubeless function call get-python-36 |egrep hello.world
 
 get-python-url-deps:
-	kubeless function deploy get-python-url-deps --runtime python2.7 --handler helloget.foo --from-file https://raw.githubusercontent.com/kubeless/kubeless/v1.0.0-alpha.1/examples/python/hellowithdeps.py --dependencies https://raw.githubusercontent.com/kubeless/kubeless/v1.0.0-alpha.1/examples/python/requirements.txt
+	cd python && tar czf hellowithdeps.tgz hellowithdeps.py hellowithdepshelper.py && cd ..
+	kubeless function deploy get-python-url-deps --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.tgz --dependencies $(BASE_URL)/examples/python/requirements.txt
 
 get-python-url-deps-verify:
 	kubeless function call get-python-url-deps |egrep Google	
 
 get-node-url-zip:
-	kubeless function deploy get-node-url-zip --runtime nodejs10 --handler index.helloGet --from-file https://github.com/kubeless/kubeless/blob/master/examples/nodejs/helloFunctions.zip?raw=true
+	kubeless function deploy get-node-url-zip --runtime nodejs10 --handler index.helloGet --from-file $(BASE_URL)/examples/nodejs/helloFunctions.zip
+
+get-node-url-tar-gz:
+	kubeless function deploy get-node-url-tar-gz --runtime nodejs10 --handler index.helloGet --from-file $(BASE_URL)/examples/nodejs/helloFunctions.tar.gz
+
+get-node-url-tar-bz2:
+	kubeless function deploy get-node-url-tar-bz2 --runtime nodejs10 --handler index.helloGet --from-file $(BASE_URL)/examples/nodejs/helloFunctions.tar.bz2
+
+get-node-url-tar-xz:
+	kubeless function deploy get-node-url-tar-xz --runtime nodejs10 --handler index.helloGet --from-file $(BASE_URL)/examples/nodejs/helloFunctions.tar.xz
 
 get-node-url-zip-verify:
 	kubeless function call get-node-url-zip |egrep hello.world
 
+get-node-url-tar-gz-verify:
+	kubeless function call get-node-url-tar-gz |egrep hello.world
+
+get-node-url-tar-bz2-verify:
+	kubeless function call get-node-url-tar-bz2 |egrep hello.world
+
+get-node-url-tar-xz-verify:
+	kubeless function call get-node-url-tar-xz |egrep hello.world
+
 scheduled-get-python:
 	kubeless function deploy scheduled-get-python --schedule "* * * * *" --runtime python2.7 --handler helloget.foo --from-file python/helloget.py
 
@@ -182,8 +225,11 @@ get-python-metadata:
 
 get-python-metadata-verify:
 	kubeless function call get-python-metadata |egrep hello.world
-	kubectl get po -o jsonpath='{.items[0].spec.containers[0].env}' -l function=get-python-metadata | grep "name:foo value:bar"
-	kubectl get po -o jsonpath='{.items[0].spec.containers[0].env}' -l function=get-python-metadata | grep "name:bar value:foo"
+	kubectl get po -o jsonpath='{.items[0].spec.containers[0].env}' -l function=get-python-metadata | grep '"name":"foo","value":"bar"'
+	kubectl get po -o jsonpath='{.items[0].spec.containers[0].env}' -l function=get-python-metadata | grep '"name":"bar","value":"foo"'
+	kubectl get po -o jsonpath='{.items[0].metadata.labels}' -l function=get-python-metadata | grep '"foo":"bar"'
+	kubectl get po -o jsonpath='{.items[0].metadata.labels}' -l function=get-python-metadata | grep '"bar":"foo"'
+	kubectl get po -o jsonpath='{.items[0].metadata.labels}' -l function=get-python-metadata | grep '"foobar":""'
 
 get-python-secrets:
 	kubectl create secret generic test-secret --from-literal=key=MY_KEY || true

kubeless-non-rbac.jsonnet

@@ -88,7 +88,7 @@ local kubelessConfig  = configMap.default("kubeless-config", namespace) +
     configMap.data({"runtime-images": std.toString(runtimesSrc)})+
     configMap.data({"enable-build-step": "false"})+
     configMap.data({"function-registry-tls-verify": "true"})+
-    configMap.data({"provision-image": "kubeless/unzip@sha256:4cbab8b7c70b0d8f10dbd920e434d39a197a62e0e55124d6b5f14e957e358258"})+
+    configMap.data({"provision-image": "kubeless/unzip@sha256:e867f9b366ffb1a25f14baf83438db426ced4f7add56137b7300d32507229b5a"})+
     configMap.data({"provision-image-secret": ""})+
     configMap.data({"builder-image": "kubeless/function-image-builder:latest"})+
     configMap.data({"builder-image-secret": ""});

pkg/apis/kubeless/v1beta1/function.go

@@ -37,7 +37,7 @@ type Function struct {
 type FunctionSpec struct {
 	Handler                 string                          `json:"handler"`               // Function handler: "file.function"
 	Function                string                          `json:"function"`              // Function file content or URL of the function
-	FunctionContentType     string                          `json:"function-content-type"` // Function file content type (plain text, base64 or zip)
+	FunctionContentType     string                          `json:"function-content-type"` // Function file content type (plain text, url, base64, zip or compressedtar)
 	Checksum                string                          `json:"checksum"`              // Checksum of the file
 	Runtime                 string                          `json:"runtime"`               // Function runtime to use
 	Timeout                 string                          `json:"timeout"`               // Maximum timeout for the function to complete its execution