Skip to content

Commit 5cd5034

Browse files
joshmarshjoshmarsh
committed
Can upload code assets to s3
1 parent 7b791ee commit 5cd5034

36 files changed

+2786
-172
lines changed

‎cmd/accounts.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"fmt"
55
"io/ioutil"
66

7-
"github.com/Optum/dce-cli/internal/api"
7+
"github.com/Optum/dce-cli/internal/util/api"
88
"github.com/spf13/cobra"
99
)
1010

‎cmd/leases.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66

77
"encoding/json"
88

9-
"github.com/Optum/dce-cli/internal/api"
9+
"github.com/Optum/dce-cli/internal/util/api"
1010
"github.com/pkg/browser"
1111
"github.com/spf13/cobra"
1212
)

‎cmd/system.go‎

Lines changed: 4 additions & 170 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,11 @@
11
package cmd
22

33
import (
4-
"context"
54
"fmt"
6-
"io"
7-
"io/ioutil"
85
"log"
9-
"math/rand"
10-
"net/http"
11-
"os"
12-
"time"
136

14-
"github.com/Optum/dce-cli/internal/terra"
15-
"github.com/shurcooL/githubv4"
7+
"github.com/Optum/dce-cli/internal/deploy"
168
"github.com/spf13/cobra"
17-
"golang.org/x/oauth2"
18-
19-
"github.com/mholt/archiver"
209
)
2110

2211
var deployNamespace string
@@ -55,172 +44,17 @@ var systemDeployCmd = &cobra.Command{
5544
Run: func(cmd *cobra.Command, args []string) {
5645

5746
log.Println("Creating terraform remote state backend infrastructure")
58-
stateBucket := createRemoteStateBackend()
47+
stateBucket := deploy.CreateRemoteStateBackend(deployNamespace)
5948

6049
log.Println("Creating DCE infrastructure")
61-
artifactsBucket := createDceInfra(stateBucket)
50+
artifactsBucket := deploy.CreateDceInfra(deployNamespace, stateBucket)
6251
log.Println("Artifacts bucket = ", artifactsBucket)
6352

6453
// Deploy code assets to DCE infra
54+
deploy.DeployCodeAssets(deployNamespace, artifactsBucket)
6555
},
6656
}
6757

68-
func createDceInfra(stateBucket string) string {
69-
workingDir, originDir := mvToTempDir("dce-")
70-
defer os.RemoveAll(workingDir)
71-
defer os.Chdir(originDir)
72-
73-
log.Println("Downloading DCE terraform modules")
74-
artifactsFileName := "terraform_artifacts.zip"
75-
downloadGithubReleaseAsset(artifactsFileName)
76-
77-
// TODO:
78-
// Protect against zip-slip vulnerability? https://snyk.io/research/zip-slip-vulnerability
79-
//
80-
// err := z.Walk("/Users/matt/Desktop/test.zip", func(f archiver.File) error {
81-
// zfh, ok := f.Header.(zip.FileHeader)
82-
// if ok {
83-
// fmt.Println("Filename:", zfh.Name)
84-
// }
85-
// return nil
86-
// })
87-
88-
err := archiver.Unarchive(artifactsFileName, ".")
89-
if err != nil {
90-
log.Fatalf("error: %v", err)
91-
}
92-
93-
os.Remove(artifactsFileName)
94-
files, err := ioutil.ReadDir("./")
95-
if len(files) != 1 || !files[0].IsDir() {
96-
log.Fatalf("Unexpected content in DCE assets archive")
97-
}
98-
os.Chdir(files[0].Name())
99-
100-
log.Println("Initializing terraform working directory")
101-
terra.Init([]string{"-backend-config=bucket=" + stateBucket, "-backend-config=key=local-tf-state"})
102-
103-
log.Println("Applying DCE infrastructure")
104-
var namesSpace string
105-
if deployNamespace != "" {
106-
namesSpace = deployNamespace
107-
} else {
108-
namesSpace = "dce-" + getRandString(6)
109-
}
110-
terra.Apply(namesSpace)
111-
112-
log.Println("Retrieving artifacts bucket name from terraform outputs")
113-
artifactsBucket := terra.GetOutput("artifacts_bucket_name")
114-
log.Println(" -->", artifactsBucket)
115-
116-
return artifactsBucket
117-
}
118-
119-
func downloadGithubReleaseAsset(assetName string) {
120-
src := oauth2.StaticTokenSource(
121-
&oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")},
122-
)
123-
oauthHTTPClient := oauth2.NewClient(context.Background(), src)
124-
125-
var query struct {
126-
Viewer struct {
127-
Login githubv4.String
128-
CreatedAt githubv4.DateTime
129-
}
130-
Repository struct {
131-
Releases struct {
132-
Nodes []struct {
133-
TagName githubv4.String
134-
ReleaseAssets struct {
135-
Nodes []struct {
136-
ID githubv4.String
137-
DownloadURL githubv4.String
138-
URL string
139-
}
140-
} `graphql:"releaseAssets(last: 1, name: \"terraform_artifacts.zip\")"`
141-
}
142-
} `graphql:"releases(last: 1)"`
143-
} `graphql:"repository(owner: \"Optum\", name: \"Redbox\")"`
144-
}
145-
146-
ghClient := githubv4.NewClient(oauthHTTPClient)
147-
err := ghClient.Query(context.Background(), &query, nil)
148-
if err != nil {
149-
log.Fatalf("error: %v", err)
150-
}
151-
fmt.Println(" Query Response:", query.Repository.Releases.Nodes[0].ReleaseAssets.Nodes[0].URL)
152-
153-
req, err := http.NewRequest("GET", query.Repository.Releases.Nodes[0].ReleaseAssets.Nodes[0].URL, nil)
154-
resp, err := http.DefaultClient.Do(req)
155-
if err != nil {
156-
log.Fatalf("error: %v", err)
157-
}
158-
defer resp.Body.Close()
159-
160-
out, err := os.Create(assetName)
161-
if err != nil {
162-
log.Fatalf("error: %v", err)
163-
}
164-
defer out.Close()
165-
_, err = io.Copy(out, resp.Body)
166-
if err != nil {
167-
log.Fatalf("error: %v", err)
168-
}
169-
}
170-
171-
func createRemoteStateBackend() string {
172-
workingDir, originDir := mvToTempDir("dce-init-")
173-
defer os.RemoveAll(workingDir)
174-
defer os.Chdir(originDir)
175-
176-
log.Println("Creating terraform remote backend template (init.tf)")
177-
fileName := workingDir + "/" + "init.tf"
178-
err := ioutil.WriteFile(fileName, []byte(terra.RemoteBackend), 0644)
179-
if err != nil {
180-
log.Fatalf("error: %v", err)
181-
}
182-
183-
log.Println("Initializing terraform working directory and building remote state infrastructure")
184-
terra.Init([]string{})
185-
if deployNamespace != "" {
186-
terra.Apply(deployNamespace)
187-
} else {
188-
terra.Apply("dce-default-" + getRandString(8))
189-
}
190-
191-
log.Println("Retrieving remote state bucket name from terraform outputs")
192-
stateBucket := terra.GetOutput("bucket")
193-
log.Println(" -->", stateBucket)
194-
195-
return stateBucket
196-
}
197-
198-
func mvToTempDir(prefix string) (string, string) {
199-
log.Println("Creating temporary terraform working directory")
200-
destinationDir, err := ioutil.TempDir("", prefix)
201-
if err != nil {
202-
log.Fatalln(err)
203-
}
204-
log.Println(" -->" + destinationDir)
205-
originDir, err := os.Getwd()
206-
if err != nil {
207-
log.Fatalln(err)
208-
}
209-
os.Chdir(destinationDir)
210-
return destinationDir, originDir
211-
}
212-
213-
// https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-go
214-
func getRandString(n int) string {
215-
rand.Seed(time.Now().UnixNano())
216-
const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
217-
b := make([]byte, n)
218-
for i := range b {
219-
b[i] = letterBytes[rand.Int63()%int64(len(letterBytes))]
220-
}
221-
return string(b)
222-
}
223-
22458
/*
22559
Logs Namespace
22660
*/

‎internal/deploy/deploy.go‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package deploy
2+
3+
import (
4+
"io/ioutil"
5+
"log"
6+
"math/rand"
7+
"os"
8+
"time"
9+
10+
"github.com/Optum/dce-cli/internal/util/awshelpers"
11+
"github.com/Optum/dce-cli/internal/util/ghub"
12+
"github.com/Optum/dce-cli/internal/util/terra"
13+
"github.com/mholt/archiver"
14+
)
15+
16+
const ArtifactsFileName = "terraform_artifacts.zip"
17+
const AssetsFileName = "build_artifacts.zip"
18+
19+
func CreateRemoteStateBackend(namespace string) string {
20+
tmpDir, originDir := mvToTempDir("dce-init-")
21+
defer os.RemoveAll(tmpDir)
22+
defer os.Chdir(originDir)
23+
24+
log.Println("Creating terraform remote backend template (init.tf)")
25+
fileName := tmpDir + "/" + "init.tf"
26+
err := ioutil.WriteFile(fileName, []byte(terra.RemoteBackend), 0644)
27+
if err != nil {
28+
log.Fatalf("error: %v", err)
29+
}
30+
31+
log.Println("Initializing terraform working directory and building remote state infrastructure")
32+
terra.Init([]string{})
33+
if namespace != "" {
34+
terra.Apply(namespace)
35+
} else {
36+
terra.Apply("dce-default-" + getRandString(8))
37+
}
38+
39+
log.Println("Retrieving remote state bucket name from terraform outputs")
40+
stateBucket := terra.GetOutput("bucket")
41+
log.Println("Remote state bucket = ", stateBucket)
42+
43+
return stateBucket
44+
}
45+
46+
func CreateDceInfra(namespace string, stateBucket string) string {
47+
tmpDir, originDir := mvToTempDir("dce-")
48+
defer os.RemoveAll(tmpDir)
49+
defer os.Chdir(originDir)
50+
51+
log.Println("Downloading DCE terraform modules")
52+
ghub.DownloadGithubReleaseAsset(ArtifactsFileName)
53+
// TODO:
54+
// Protect against zip-slip vulnerability? https://snyk.io/research/zip-slip-vulnerability
55+
//
56+
// err := z.Walk("/Users/matt/Desktop/test.zip", func(f archiver.File) error {
57+
// zfh, ok := f.Header.(zip.FileHeader)
58+
// if ok {
59+
// fmt.Println("Filename:", zfh.Name)
60+
// }
61+
// return nil
62+
// })
63+
err := archiver.Unarchive(ArtifactsFileName, ".")
64+
if err != nil {
65+
log.Fatalf("error: %v", err)
66+
}
67+
os.Remove(ArtifactsFileName)
68+
files, err := ioutil.ReadDir("./")
69+
if len(files) != 1 || !files[0].IsDir() {
70+
log.Fatalf("Unexpected content in DCE assets archive")
71+
}
72+
os.Chdir(files[0].Name())
73+
74+
log.Println("Initializing terraform working directory")
75+
terra.Init([]string{"-backend-config=bucket=" + stateBucket, "-backend-config=key=local-tf-state"})
76+
77+
log.Println("Applying DCE infrastructure")
78+
if namespace != "" {
79+
terra.Apply(namespace)
80+
} else {
81+
terra.Apply("dce-" + getRandString(6))
82+
}
83+
84+
log.Println("Retrieving artifacts bucket name from terraform outputs")
85+
artifactsBucket := terra.GetOutput("artifacts_bucket_name")
86+
log.Println("artifacts bucket name = ", artifactsBucket)
87+
88+
return artifactsBucket
89+
}
90+
91+
func DeployCodeAssets(deployNamespace string, artifactsBucket string) {
92+
tmpDir, originDir := mvToTempDir("dce-")
93+
defer os.RemoveAll(tmpDir)
94+
defer os.Chdir(originDir)
95+
96+
log.Println("Downloading DCE code assets")
97+
ghub.DownloadGithubReleaseAsset(AssetsFileName)
98+
// TODO:
99+
// Protect against zip-slip vulnerability? https://snyk.io/research/zip-slip-vulnerability
100+
//
101+
// err := z.Walk("/Users/matt/Desktop/test.zip", func(f archiver.File) error {
102+
// zfh, ok := f.Header.(zip.FileHeader)
103+
// if ok {
104+
// fmt.Println("Filename:", zfh.Name)
105+
// }
106+
// return nil
107+
// })
108+
err := archiver.Unarchive(AssetsFileName, ".")
109+
if err != nil {
110+
log.Fatalf("error: %v", err)
111+
}
112+
os.Remove(AssetsFileName)
113+
files, err := ioutil.ReadDir("./")
114+
if len(files) != 2 || !files[0].IsDir() || !files[1].IsDir() {
115+
log.Fatalf("Unexpected content in DCE assets archive")
116+
}
117+
// os.Chdir(files[0].Name())
118+
119+
//LEFT OFF HERE, deploy to lambdas and stuff
120+
121+
// 1. Upload lambda and codebuild zips to s3
122+
awshelpers.UploadDirectoryToS3(".", artifactsBucket, "")
123+
124+
// 2. Point lambdas at the code in s3
125+
126+
// 3. Publish new lambda versions
127+
}
128+
129+
func mvToTempDir(prefix string) (string, string) {
130+
log.Println("Creating temporary working directory")
131+
destinationDir, err := ioutil.TempDir("", prefix)
132+
if err != nil {
133+
log.Fatalln(err)
134+
}
135+
log.Println(" -->" + destinationDir)
136+
originDir, err := os.Getwd()
137+
if err != nil {
138+
log.Fatalln(err)
139+
}
140+
os.Chdir(destinationDir)
141+
return destinationDir, originDir
142+
}
143+
144+
// https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-go
145+
func getRandString(n int) string {
146+
rand.Seed(time.Now().UnixNano())
147+
const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
148+
b := make([]byte, n)
149+
for i := range b {
150+
b[i] = letterBytes[rand.Int63()%int64(len(letterBytes))]
151+
}
152+
return string(b)
153+
}

internal/api/client.go renamed to internal/util/api/client.go

File renamed without changes.

0 commit comments

Comments
 (0)