diff --git a/.circleci/config.yml b/.circleci/config.yml index 21d4a7b93..0ed97c2ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,6 +27,7 @@ exports: &exports echo "export FUNCTION_IMAGE_BUILDER=${BUILDER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV echo "export KUBECFG_JPATH=/home/circleci/src/github.com/kubeless/kubeless/ksonnet-lib" >> $BASH_ENV echo "export PATH=$(pwd)/bats/libexec:$GOPATH/bin:$PATH" >> $BASH_ENV + echo "export GIT_SHA1=${CIRCLE_SHA1}" >> $BASH_ENV restore_workspace: &restore_workspace run: | make bootstrap @@ -112,6 +113,8 @@ jobs: image: ubuntu-1604:201903-01 steps: - checkout + - run: sudo apt-get update -y + - run: sudo apt-get install -y tar gzip bzip2 xz-utils - attach_workspace: at: /tmp/go - <<: *exports diff --git a/cmd/kubeless/function/function_test.go b/cmd/kubeless/function/function_test.go index fe769ff4f..a92ed9709 100644 --- a/cmd/kubeless/function/function_test.go +++ b/cmd/kubeless/function/function_test.go @@ -17,7 +17,11 @@ limitations under the License. package function import ( + "archive/tar" "archive/zip" + "compress/gzip" + "crypto/sha256" + "encoding/hex" "fmt" "io" "io/ioutil" @@ -331,26 +335,52 @@ func TestGetFunctionDescription(t *testing.T) { t.Errorf("Unexpected result. Expecting:\n %+v\n Received %+v\n", newFunction, *result3) } - // It should detect that it is a Zip file + // It should detect that it is a Zip file or a compressed tar file file, err = os.Open(file.Name()) if err != nil { t.Error(err) } - newfile, err := os.Create(file.Name() + ".zip") + + zipFile, err := os.Create(file.Name() + ".zip") + if err != nil { + t.Error(err) + } + defer os.Remove(zipFile.Name()) // clean up + + tarGzFile, err := os.Create(file.Name() + ".tar.gz") if err != nil { t.Error(err) } - defer os.Remove(newfile.Name()) // clean up - zipW := zip.NewWriter(newfile) + defer os.Remove(tarGzFile.Name()) // clean up + + zipW := zip.NewWriter(zipFile) + gzipW := gzip.NewWriter(tarGzFile) + tarW := tar.NewWriter(gzipW) + info, err := file.Stat() if err != nil { t.Error(err) } - header, err := zip.FileInfoHeader(info) + + zipHeader, err := zip.FileInfoHeader(info) + if err != nil { + t.Error(err) + } + writer, err := zipW.CreateHeader(zipHeader) + if err != nil { + t.Error(err) + } + _, err = io.Copy(writer, file) + if err != nil { + t.Error(err) + } + + tarHeader, err := tar.FileInfoHeader(info, info.Name()) if err != nil { t.Error(err) } - writer, err := zipW.CreateHeader(header) + tarHeader.Name = file.Name() + err = tarW.WriteHeader(tarHeader) if err != nil { t.Error(err) } @@ -358,15 +388,28 @@ func TestGetFunctionDescription(t *testing.T) { if err != nil { t.Error(err) } + file.Close() zipW.Close() + zipFile.Close() + tarW.Close() + gzipW.Close() + tarGzFile.Close() - result4, err := getFunctionDescription("test", "default", "file.handler", newfile.Name(), "dependencies", "runtime", "", "", "", "", "Always", "", 8080, 0, false, []string{}, []string{}, []string{}, []string{}, expectedFunction) + result4A, err := getFunctionDescription("test", "default", "file.handler", zipFile.Name(), "dependencies", "runtime", "", "", "", "", "Always", "", 8080, 0, false, []string{}, []string{}, []string{}, []string{}, expectedFunction) if err != nil { t.Error(err) } - if result4.Spec.FunctionContentType != "base64+zip" { - t.Errorf("Should return base64+zip, received %s", result4.Spec.FunctionContentType) + if result4A.Spec.FunctionContentType != "base64+zip" { + t.Errorf("Should return base64+zip, received %s", result4A.Spec.FunctionContentType) + } + + result4B, err := getFunctionDescription("test", "default", "file.handler", tarGzFile.Name(), "dependencies", "runtime", "", "", "", "", "Always", "", 8080, 0, false, []string{}, []string{}, []string{}, []string{}, expectedFunction) + if err != nil { + t.Error(err) + } + if result4B.Spec.FunctionContentType != "base64+compressedtar" { + t.Errorf("Should return base64+compressedtar, received %s", result4B.Spec.FunctionContentType) } // It should maintain previous HPA definition @@ -496,30 +539,65 @@ func TestGetFunctionDescription(t *testing.T) { if !reflect.DeepEqual(expectedURLFunction, *result7) { t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result7) } - // end test - // it should handle zip files from a URL and detect url+zip encoding - zipBytes, err := ioutil.ReadFile(newfile.Name()) + // It should handle zip files and compressed tar files from a URL and detect url+zip and url+compressedtar encoding respectively + zipBytes, err := ioutil.ReadFile(zipFile.Name()) if err != nil { t.Error(err) } - ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ts2A := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write(zipBytes) })) - defer ts2.Close() + defer ts2A.Close() expectedURLFunction.Spec.FunctionContentType = "url+zip" - expectedURLFunction.Spec.Function = ts2.URL + "/test.zip" - result8, err := getFunctionDescription("test", "default", "file.handler", ts2.URL+"/test.zip", "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", "serviceAccount", 8080, 0, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, []string{}, kubelessApi.Function{}) + expectedURLFunction.Spec.Function = ts2A.URL + "/test.zip" + expectedURLFunction.Spec.Checksum, err = getSha256(zipBytes) + if err != nil { + t.Error(err) + } + + result8A, err := getFunctionDescription("test", "default", "file.handler", ts2A.URL+"/test.zip", "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", "serviceAccount", 8080, 0, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, []string{"foo3=bar3", "baz3:qux3"}, kubelessApi.Function{}) if err != nil { t.Error(err) } - if result8.Spec.FunctionContentType != "url+zip" { - t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8) + if !reflect.DeepEqual(expectedURLFunction, *result8A) { + t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8A) } - if result8.Spec.Function != ts2.URL+"/test.zip" { - t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8) + + tarGzBytes, err := ioutil.ReadFile(tarGzFile.Name()) + if err != nil { + t.Error(err) + } + ts2B := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(tarGzBytes) + })) + defer ts2B.Close() + + expectedURLFunction.Spec.FunctionContentType = "url+compressedtar" + expectedURLFunction.Spec.Function = ts2B.URL + "/test.tar.gz" + expectedURLFunction.Spec.Checksum, err = getSha256(tarGzBytes) + if err != nil { + t.Error(err) + } + + result8B, err := getFunctionDescription("test", "default", "file.handler", ts2B.URL+"/test.tar.gz", "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", "serviceAccount", 8080, 0, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, []string{"foo3=bar3", "baz3:qux3"}, kubelessApi.Function{}) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(expectedURLFunction, *result8B) { + t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8B) } // end test } + +func getSha256(bytes []byte) (string, error) { + h := sha256.New() + _, err := h.Write(bytes) + if err != nil { + return "", err + } + checksum := hex.EncodeToString(h.Sum(nil)) + return "sha256:" + checksum, nil +} diff --git a/docker/unzip/Dockerfile b/docker/unzip/Dockerfile index cd4c4891c..8f14a79af 100644 --- a/docker/unzip/Dockerfile +++ b/docker/unzip/Dockerfile @@ -1,2 +1,2 @@ FROM bitnami/minideb -RUN install_packages unzip curl ca-certificates +RUN install_packages unzip curl ca-certificates tar gzip bzip2 xz-utils diff --git a/docs/advanced-function-deployment.md b/docs/advanced-function-deployment.md index bb07a9c12..26fce8d68 100644 --- a/docs/advanced-function-deployment.md +++ b/docs/advanced-function-deployment.md @@ -27,10 +27,10 @@ The fields that a Function specification can contain are: - Runtime: Runtime ID and version that the function will use. It should match one of the availables in the [Kubeless configuration](/docs/function-controller-configuration). - Timeout: Maximum timeout for the given function. After that time, the function execution will be terminated. - - Handler: Pair of `.`. When using `zip` in `function-content-type` the `` will be used to find the file with the function to expose. In other case it will be used just as a final file name. `` is used to select the function to run from the exported functions of ``. This field is mandatory and should match with an exported function. + - Handler: Pair of `.`. When using `zip` or `compressedtar` in `function-content-type`, the `` will be used to find the file with the function to expose. In other cases, it will be used just as a final file name. `` is used to select the function to run from the exported functions of ``. This field is mandatory and should match with an exported function. - Deps: Dependencies of the function. The format of this field will depend on the runtime, e.g. a `package.json` for NodeJS functions or a `Gemfile` for Ruby. - Checksum: SHA256 of the function content. - - Function content type: Content type of the function. Current supported values are `base64`, `url` or `text`. If the content is zipped the suffix `+zip` should be added. + - Function content type: Content type of the function. Current supported values are `base64`, `url` or `text`. If the content is zipped, the suffix `+zip` should be added. If the content is a gzip/bzip2/xz compressed tar file, the suffix `+compressedtar` should be added. - Function: Function content. Apart from the basic parameters, it is possible to add the specification of a `Deployment`, a `Service` or an `Horizontal Pod Autoscaler` that Kubeless will use to generate them. diff --git a/docs/function-controller-configuration.md b/docs/function-controller-configuration.md index c063e2cd8..d2d536d23 100644 --- a/docs/function-controller-configuration.md +++ b/docs/function-controller-configuration.md @@ -235,7 +235,7 @@ It is possible to configure the different images that Kubeless uses for deploy a - Runtime Image: Image used to execute the function. - Init Image: Image used for installing the function and/or dependencies. - (Optional) Image Pull Secrets: Secret required to pull the image in case the repository is private. - - The image used to populate the base image with the function. This is called `provision-image`. This image should have at least `unzip` and `curl`. It is also possible to specify `provision-image-secret` to specify a secret to pull that image from a private registry. + - The image used to populate the base image with the function. This is called `provision-image`. This image should have at least `unzip`, `GNU tar`, `gzip`, `bzip2`, `xz` and `curl`. It is also possible to specify `provision-image-secret` to specify a secret to pull that image from a private registry. - The image used to build function images. This is called `builder-image`. This image is optional since its usage can be disabled with the property `enable-build-step`. A Dockerfile to build this image can be found [here](https://github.com/kubeless/kubeless/tree/master/docker/function-image-builder). It is also possible to specify `builder-image-secret` to specify a secret to pull that image from a private registry. ## Authenticate Kubeless Function Controller using OAuth Bearer Token diff --git a/docs/quick-start.md b/docs/quick-start.md index 01f261c94..57fa97460 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -89,7 +89,7 @@ Let's dissect the command: * `hello`: This is the name of the function we want to deploy. * `--runtime python2.7`: This is the runtime we want to use to run our function. Available runtimes can be found executing `kubeless get-server-config`. -* `--from-file test.py`: This is the file containing the function code. It is supported to specify a zip file as far as it doesn't exceed the maximum size for an etcd entry (1 MB). +* `--from-file test.py`: This is the file containing the function code. Specifying a zip file or a gzip/bzip2/xz compressed tar file (see [list of supported suffixes](https://en.wikipedia.org/wiki/Tar_(computing)#Suffixes_for_compressed_files) for compressed tar files) is supported as long as it doesn't exceed the maximum size for an etcd entry (1 MB). * `--handler test.hello`: This specifies the file and the exposed function that will be used when receiving requests. In this example we are using the function `hello` from the file `test.py`. You can find the rest of options available when deploying a function executing `kubeless function deploy --help` diff --git a/examples/Makefile b/examples/Makefile index 11a226457..315395566 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -1,3 +1,6 @@ +GIT_SHA1 ?= master +BASE_URL := https://raw.githubusercontent.com/kubeless/kubeless/$(GIT_SHA1) + get-python: kubeless function deploy get-python --runtime python2.7 --handler helloget.foo --from-file python/helloget.py @@ -15,12 +18,33 @@ get-python-update-verify: kubeless function call get-python |egrep hello.world.updated get-python-deps: - cd python && zip hellowithdeps.zip hellowithdeps.py hellowithdepshelper.py && cd .. + cd python && zip hellowithdeps.zip hellowithdeps.py hellowithdepshelper.py && cd .. kubeless function deploy get-python-deps --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.zip --dependencies python/requirements.txt +get-python-deps-tar-gz: + cd python && tar czf hellowithdeps.tar.gz hellowithdeps.py hellowithdepshelper.py && cd .. + kubeless function deploy get-python-deps-tar-gz --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.tar.gz --dependencies python/requirements.txt + +get-python-deps-tar-bz2: + cd python && tar cjf hellowithdeps.tar.bz2 hellowithdeps.py hellowithdepshelper.py && cd .. + kubeless function deploy get-python-deps-tar-bz2 --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.tar.bz2 --dependencies python/requirements.txt + +get-python-deps-tar-xz: + cd python && tar cJf hellowithdeps.tar.xz hellowithdeps.py hellowithdepshelper.py && cd .. + kubeless function deploy get-python-deps-tar-xz --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.tar.xz --dependencies python/requirements.txt + get-python-deps-verify: kubeless function call get-python-deps |egrep Google +get-python-deps-tar-gz-verify: + kubeless function call get-python-deps-tar-gz |egrep Google + +get-python-deps-tar-bz2-verify: + kubeless function call get-python-deps-tar-bz2 |egrep Google + +get-python-deps-tar-xz-verify: + kubeless function call get-python-deps-tar-xz |egrep Google + get-python-custom-port: kubeless function deploy get-python-custom-port --runtime python2.7 --handler helloget.foo --from-file python/helloget.py --port 8081 @@ -52,17 +76,36 @@ get-python-36-verify: kubeless function call get-python-36 |egrep hello.world get-python-url-deps: - kubeless function deploy get-python-url-deps --runtime python2.7 --handler helloget.foo --from-file https://raw.githubusercontent.com/kubeless/kubeless/v1.0.0-alpha.1/examples/python/hellowithdeps.py --dependencies https://raw.githubusercontent.com/kubeless/kubeless/v1.0.0-alpha.1/examples/python/requirements.txt + cd python && tar czf hellowithdeps.tgz hellowithdeps.py hellowithdepshelper.py && cd .. + kubeless function deploy get-python-url-deps --runtime python2.7 --handler hellowithdeps.foo --from-file python/hellowithdeps.tgz --dependencies $(BASE_URL)/examples/python/requirements.txt get-python-url-deps-verify: kubeless function call get-python-url-deps |egrep Google get-node-url-zip: - kubeless function deploy get-node-url-zip --runtime nodejs10 --handler index.helloGet --from-file https://github.com/kubeless/kubeless/blob/master/examples/nodejs/helloFunctions.zip?raw=true + kubeless function deploy get-node-url-zip --runtime nodejs10 --handler index.helloGet --from-file $(BASE_URL)/examples/nodejs/helloFunctions.zip + +get-node-url-tar-gz: + kubeless function deploy get-node-url-tar-gz --runtime nodejs10 --handler index.helloGet --from-file $(BASE_URL)/examples/nodejs/helloFunctions.tar.gz + +get-node-url-tar-bz2: + kubeless function deploy get-node-url-tar-bz2 --runtime nodejs10 --handler index.helloGet --from-file $(BASE_URL)/examples/nodejs/helloFunctions.tar.bz2 + +get-node-url-tar-xz: + kubeless function deploy get-node-url-tar-xz --runtime nodejs10 --handler index.helloGet --from-file $(BASE_URL)/examples/nodejs/helloFunctions.tar.xz get-node-url-zip-verify: kubeless function call get-node-url-zip |egrep hello.world +get-node-url-tar-gz-verify: + kubeless function call get-node-url-tar-gz |egrep hello.world + +get-node-url-tar-bz2-verify: + kubeless function call get-node-url-tar-bz2 |egrep hello.world + +get-node-url-tar-xz-verify: + kubeless function call get-node-url-tar-xz |egrep hello.world + scheduled-get-python: kubeless function deploy scheduled-get-python --schedule "* * * * *" --runtime python2.7 --handler helloget.foo --from-file python/helloget.py @@ -182,8 +225,11 @@ get-python-metadata: get-python-metadata-verify: kubeless function call get-python-metadata |egrep hello.world - kubectl get po -o jsonpath='{.items[0].spec.containers[0].env}' -l function=get-python-metadata | grep "name:foo value:bar" - kubectl get po -o jsonpath='{.items[0].spec.containers[0].env}' -l function=get-python-metadata | grep "name:bar value:foo" + kubectl get po -o jsonpath='{.items[0].spec.containers[0].env}' -l function=get-python-metadata | grep '"name":"foo","value":"bar"' + kubectl get po -o jsonpath='{.items[0].spec.containers[0].env}' -l function=get-python-metadata | grep '"name":"bar","value":"foo"' + kubectl get po -o jsonpath='{.items[0].metadata.labels}' -l function=get-python-metadata | grep '"foo":"bar"' + kubectl get po -o jsonpath='{.items[0].metadata.labels}' -l function=get-python-metadata | grep '"bar":"foo"' + kubectl get po -o jsonpath='{.items[0].metadata.labels}' -l function=get-python-metadata | grep '"foobar":""' get-python-secrets: kubectl create secret generic test-secret --from-literal=key=MY_KEY || true diff --git a/examples/nodejs/helloFunctions.tar.bz2 b/examples/nodejs/helloFunctions.tar.bz2 new file mode 100644 index 000000000..db59f839d Binary files /dev/null and b/examples/nodejs/helloFunctions.tar.bz2 differ diff --git a/examples/nodejs/helloFunctions.tar.gz b/examples/nodejs/helloFunctions.tar.gz new file mode 100644 index 000000000..d85c623ed Binary files /dev/null and b/examples/nodejs/helloFunctions.tar.gz differ diff --git a/examples/nodejs/helloFunctions.tar.xz b/examples/nodejs/helloFunctions.tar.xz new file mode 100644 index 000000000..7cf3c3ef7 Binary files /dev/null and b/examples/nodejs/helloFunctions.tar.xz differ diff --git a/kubeless-non-rbac.jsonnet b/kubeless-non-rbac.jsonnet index ddd83e751..ca620f79b 100644 --- a/kubeless-non-rbac.jsonnet +++ b/kubeless-non-rbac.jsonnet @@ -88,7 +88,7 @@ local kubelessConfig = configMap.default("kubeless-config", namespace) + configMap.data({"runtime-images": std.toString(runtimesSrc)})+ configMap.data({"enable-build-step": "false"})+ configMap.data({"function-registry-tls-verify": "true"})+ - configMap.data({"provision-image": "kubeless/unzip@sha256:4cbab8b7c70b0d8f10dbd920e434d39a197a62e0e55124d6b5f14e957e358258"})+ + configMap.data({"provision-image": "kubeless/unzip@sha256:e867f9b366ffb1a25f14baf83438db426ced4f7add56137b7300d32507229b5a"})+ configMap.data({"provision-image-secret": ""})+ configMap.data({"builder-image": "kubeless/function-image-builder:latest"})+ configMap.data({"builder-image-secret": ""}); diff --git a/pkg/apis/kubeless/v1beta1/function.go b/pkg/apis/kubeless/v1beta1/function.go index 2b34d1e53..84745d3e9 100644 --- a/pkg/apis/kubeless/v1beta1/function.go +++ b/pkg/apis/kubeless/v1beta1/function.go @@ -37,7 +37,7 @@ type Function struct { type FunctionSpec struct { Handler string `json:"handler"` // Function handler: "file.function" Function string `json:"function"` // Function file content or URL of the function - FunctionContentType string `json:"function-content-type"` // Function file content type (plain text, base64 or zip) + FunctionContentType string `json:"function-content-type"` // Function file content type (plain text, url, base64, zip or compressedtar) Checksum string `json:"checksum"` // Checksum of the file Runtime string `json:"runtime"` // Function runtime to use Timeout string `json:"timeout"` // Maximum timeout for the function to complete its execution diff --git a/pkg/utils/kubelessutil.go b/pkg/utils/kubelessutil.go index 577e94d6a..903712a3d 100644 --- a/pkg/utils/kubelessutil.go +++ b/pkg/utils/kubelessutil.go @@ -111,11 +111,17 @@ func getProvisionContainer(function, checksum, fileName, handler, contentType, r } } - // Extract content in case it is a Zip file if strings.Contains(contentType, "zip") { + // Extract content in case it is a Zip file prepareCommand = appendToCommand(prepareCommand, fmt.Sprintf("unzip -o %s -d %s", originFile, runtimeVolume.MountPath), ) + } else if strings.Contains(contentType, "compressedtar") { + // Extract content in case it is a compressed tar file. + // The `tar` command auto-detects the compression type. + prepareCommand = appendToCommand(prepareCommand, + fmt.Sprintf("tar xf %s -C %s", originFile, runtimeVolume.MountPath), + ) } else { // Copy the target as a single file destFileName, err := getFileName(handler, contentType, runtime, lr) @@ -355,9 +361,6 @@ func populatePodSpec(funcObj *kubelessApi.Function, lr *langruntime.Langruntimes if err != nil { return err } - if err != nil { - return err - } srcVolumeMount := v1.VolumeMount{ Name: depsVolumeName, MountPath: "/src", @@ -879,15 +882,28 @@ func DryRunFmt(format string, trigger interface{}) (string, error) { } } +// getCompressionType returns the compression type (if any) of the given file by looking at the file extension +func getCompressionType(filename string) (compressionType string) { + if strings.HasSuffix(filename, ".zip") { + compressionType = "+zip" + } + + extensions := []string{".tar.gz", ".taz", ".tgz", ".tar.bz2", ".tb2", ".tbz", ".tbz2", ".tz2", ".tar.xz"} + for _, ext := range extensions { + if strings.HasSuffix(filename, ext) { + compressionType = "+compressedtar" + break + } + } + return +} + // GetContentType Gets the content type of a given filename func GetContentType(filename string) (string, error) { var contentType string if strings.Index(filename, "http://") == 0 || strings.Index(filename, "https://") == 0 { - contentType = "url" - if path.Ext(strings.Split(filename, "?")[0]) == ".zip" { - contentType += "+zip" - } + contentType = "url" + getCompressionType(strings.Split(filename, "?")[0]) } else { fbytes, err := ioutil.ReadFile(filename) if err != nil { @@ -898,10 +914,8 @@ func GetContentType(filename string) (string, error) { contentType = "text" } else { contentType = "base64" - if path.Ext(filename) == ".zip" { - contentType += "+zip" - } } + contentType += getCompressionType(filename) } return contentType, nil } diff --git a/pkg/utils/kubelessutil_test.go b/pkg/utils/kubelessutil_test.go index 0cc462084..5cfb1eb92 100644 --- a/pkg/utils/kubelessutil_test.go +++ b/pkg/utils/kubelessutil_test.go @@ -184,8 +184,11 @@ func TestEnsureFileNames(t *testing.T) { {name: "base64", contentType: "base64", fileNameSuffix: ".py"}, {name: "url", contentType: "url", fileNameSuffix: ".py"}, {name: "text+zip", contentType: "text+zip", fileNameSuffix: ""}, + {name: "text+compressedtar", contentType: "text+compressedtar", fileNameSuffix: ""}, {name: "base64+zip", contentType: "base64+zip", fileNameSuffix: ""}, + {name: "base64+compressedtar", contentType: "base64+compressedtar", fileNameSuffix: ""}, {name: "url+zip", contentType: "url+zip", fileNameSuffix: ""}, + {name: "url+compressedtar", contentType: "url+compressedtar", fileNameSuffix: ""}, } for _, test := range tests { @@ -960,10 +963,28 @@ func TestGetProvisionContainer(t *testing.T) { // It should extract the file in case it is a Zip c, err = getProvisionContainer("Zm9vYmFyCg==", "sha256:abc1234", "test.zip", "test.foo", "base64+zip", "python2.7", "unzip", rvol, dvol, resources, lr) + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + if !strings.HasPrefix(c.Args[0], "base64 -d < /deps/test.zip > /tmp/func.decoded") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } if !strings.Contains(c.Args[0], "unzip -o /tmp/func.decoded -d /runtime") { t.Errorf("Unexpected command: %s", c.Args[0]) } + // It should extract the compressed tar file + c, err = getProvisionContainer("Zm9vYmFyCg==", "sha256:abc1234", "test.tar.gz", "test.foo", "base64+compressedtar", "python2.7", "unzip", rvol, dvol, resources, lr) + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + if !strings.HasPrefix(c.Args[0], "base64 -d < /deps/test.tar.gz > /tmp/func.decoded") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } + if !strings.Contains(c.Args[0], "tar xf /tmp/func.decoded -C /runtime") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } + // If the content type is url it should use curl c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url", "python2.7", "unzip", rvol, dvol, resources, lr) if err != nil { @@ -973,12 +994,27 @@ func TestGetProvisionContainer(t *testing.T) { t.Errorf("Unexpected command: %s", c.Args[0]) } - // If the content type is url it should use curl - c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url+zip", "python2.7", "unzip", rvol, dvol, resources, lr) + // If the content type is url+zip it should use curl and unzip + c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.zip", "sha256:abc1234", "", "test.foo", "url+zip", "python2.7", "unzip", rvol, dvol, resources, lr) if err != nil { t.Errorf("Unexpected error: %s", err) } - if !strings.HasPrefix(c.Args[0], "curl 'https://raw.githubusercontent.com/test/test/test/test.py' -L --silent --output /tmp/func.fromurl") { + if !strings.HasPrefix(c.Args[0], "curl 'https://raw.githubusercontent.com/test/test/test/test.zip' -L --silent --output /tmp/func.fromurl") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } + if !strings.Contains(c.Args[0], "unzip -o /tmp/func.fromurl -d /runtime") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } + + // If the content type is url+compressedtar it should use curl and tar + c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.tar.gz", "sha256:abc1234", "", "test.foo", "url+compressedtar", "python2.7", "unzip", rvol, dvol, resources, lr) + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + if !strings.HasPrefix(c.Args[0], "curl 'https://raw.githubusercontent.com/test/test/test/test.tar.gz' -L --silent --output /tmp/func.fromurl") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } + if !strings.Contains(c.Args[0], "tar xf /tmp/func.fromurl -C /runtime") { t.Errorf("Unexpected command: %s", c.Args[0]) } } diff --git a/tests/integration-tests.bats b/tests/integration-tests.bats index afb6b7797..59f08a5ff 100644 --- a/tests/integration-tests.bats +++ b/tests/integration-tests.bats @@ -20,6 +20,9 @@ load ../script/libtest @test "Deploy functions to evaluate" { deploy_function get-python deploy_function get-python-deps + deploy_function get-python-deps-tar-gz + deploy_function get-python-deps-tar-bz2 + deploy_function get-python-deps-tar-xz deploy_function get-python-custom-port deploy_function timeout-nodejs deploy_function get-nodejs-multi @@ -29,6 +32,9 @@ load ../script/libtest deploy_function custom-get-python deploy_function get-python-url-deps deploy_function get-node-url-zip + deploy_function get-node-url-tar-gz + deploy_function get-node-url-tar-bz2 + deploy_function get-node-url-tar-xz } @test "Test function: get-python" { verify_function get-python @@ -36,6 +42,18 @@ load ../script/libtest @test "Test function: get-python-deps" { verify_function get-python-deps } +@test "Test function: get-python-deps-tar-gz" { + verify_function get-python-deps-tar-gz + kubeless_function_delete get-python-deps-tar-gz +} +@test "Test function: get-python-deps-tar-bz2" { + verify_function get-python-deps-tar-bz2 + kubeless_function_delete get-python-deps-tar-bz2 +} +@test "Test function: get-python-deps-tar-xz" { + verify_function get-python-deps-tar-xz + kubeless_function_delete get-python-deps-tar-xz +} @test "Test function: get-python-custom-port" { verify_function get-python-custom-port } @@ -92,4 +110,16 @@ load ../script/libtest verify_function get-node-url-zip kubeless_function_delete get-node-url-zip } +@test "Test function: get-node-url-tar-gz" { + verify_function get-node-url-tar-gz + kubeless_function_delete get-node-url-tar-gz +} +@test "Test function: get-node-url-tar-bz2" { + verify_function get-node-url-tar-bz2 + kubeless_function_delete get-node-url-tar-bz2 +} +@test "Test function: get-node-url-tar-xz" { + verify_function get-node-url-tar-xz + kubeless_function_delete get-node-url-tar-xz +} # vim: ts=2 sw=2 si et syntax=sh