Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ pipelines.
More information about Diode can be found
at [https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/](https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/).

## Prerequisites
- Go 1.24 or later installed

## Installation

```bash
Expand Down Expand Up @@ -155,6 +158,39 @@ if err != nil {
}
```

### CLI to replay dry-run files

A small helper binary is included to ingest JSON files created by the
`DryRunClient` and send them to a running Diode service.

Install the helper using `go install`:

```bash
go install github.com/netboxlabs/diode-sdk-go/cmd/diode-replay-dryrun@latest
```

This installs the command separately from the SDK library obtained with
`go get`.

Run it by providing one or more JSON files and connection details. Use `-file`
multiple times to ingest several dry-run files in a single request:

```bash
diode-replay-dryrun \
-file /tmp/example-app_1750106879725947344.json \
-file /tmp/other.json \
-target grpc://localhost:8080/diode \
-app-name example-app \
-app-version 0.1.0 \
-client-id YOUR_CLIENT_ID \
-client-secret YOUR_CLIENT_SECRET
```

The flags `-file`, `-target`, `-app-name`, and `-app-version` are required. You may
repeat `-file` to specify multiple files. OAuth2
credentials can be supplied using `-client-id` and `-client-secret` or the
`DIODE_CLIENT_ID` and `DIODE_CLIENT_SECRET` environment variables.

## Supported entities (object types)

* ASN
Expand Down
82 changes: 82 additions & 0 deletions cmd/diode-replay-dryrun/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package main

import (
"context"
"flag"
"log"
"os"
"strings"

"github.com/netboxlabs/diode-sdk-go/diode"
)

type fileList []string

func (f *fileList) String() string { return strings.Join(*f, ",") }

func (f *fileList) Set(s string) error {
*f = append(*f, s)
return nil
}

func main() {
var files fileList
flag.Var(&files, "file", "Dry-run JSON file to ingest (may be repeated)")
target := flag.String("target", "", "gRPC target of the Diode server, e.g. grpc://localhost:8080/diodet")
app := flag.String("app-name", "", "Application name used when ingesting the dry-run messages")
version := flag.String("app-version", "", "Application version used when ingesting the dry-run messages")
clientID := flag.String("client-id", "", "OAuth2 client ID. Defaults to the DIODE_CLIENT_ID environment variable if not provided")
clientSecret := flag.String("client-secret", "", "OAuth2 client secret. Defaults to the DIODE_CLIENT_SECRET environment variable if not provided")
flag.Parse()

// Fall back to environment variables if flags are not provided
if *clientID == "" {
*clientID = os.Getenv("DIODE_CLIENT_ID")
}
if *clientSecret == "" {
*clientSecret = os.Getenv("DIODE_CLIENT_SECRET")
}

if len(files) == 0 || *target == "" || *app == "" || *version == "" {
flag.Usage()
os.Exit(1)
}

client, err := diode.NewClient(
*target,
*app,
*version,
diode.WithClientID(*clientID),
diode.WithClientSecret(*clientSecret),
)
if err != nil {
log.Fatal(err)
}
defer func() {
if err := client.Close(); err != nil {
log.Printf("Failed to close client: %v", err)
}
}()

ctx := context.Background()
for _, f := range files {
entities, err := diode.LoadDryRunEntities(f)
if err != nil {
log.Printf("Failed to load %s: %v", f, err)
continue
}

resp, err := client.IngestProto(ctx, entities)
if err != nil {
log.Printf("Failed to ingest %s: %v", f, err)
continue
}

if resp.GetErrors() != nil {
log.Printf("Errors while ingesting %s: %v", f, resp.GetErrors())
continue
}

log.Printf("Ingested %d entities from %s successfully", len(entities), f)
}
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ require (
github.com/envoyproxy/protoc-gen-validate v1.0.4
github.com/google/uuid v1.6.0
github.com/stretchr/testify v1.9.0
golang.org/x/net v0.38.0
google.golang.org/grpc v1.63.2
google.golang.org/protobuf v1.33.0
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/sys v0.31.0 // indirect
golang.org/x/text v0.23.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
Expand Down
Loading