From 787d36ba1d4d9e6668f6907bbc9832d177c39226 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Wed, 17 May 2023 10:11:20 +0100 Subject: [PATCH 01/12] Upgrade to Go 1.20 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index aa4b41a..a4ef303 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/NickGowdy/deveui-cli -go 1.19 +go 1.20 require github.com/joho/godotenv v1.5.1 // direct From 984c1ae921bb4005c3ca2bf0244e038488bca413 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Wed, 17 May 2023 10:11:35 +0100 Subject: [PATCH 02/12] Optimise dockerfile --- dockerfile | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/dockerfile b/dockerfile index 467cc91..72a4577 100644 --- a/dockerfile +++ b/dockerfile @@ -1,20 +1,15 @@ -FROM golang:1.19-alpine - -RUN apk add --no-cache git - -# Set the Current Working Directory inside the container -WORKDIR /app/deveui-cli - -# We want to populate the module cache based on the go.{mod,sum} files. -COPY go.mod . - +FROM golang:1.20-alpine as base +WORKDIR /app +COPY go.* ./ RUN go mod download +COPY . ./ -COPY . . - -# Build the Go app -RUN go build -o ./deveui-cli . - +FROM base AS go-builder +ENV CGO_ENABLED=0 \ + GOOS=linux \ + GOARCH=amd64 +RUN go build -ldflags "-w -s" -o "./deveui-cli" main.go -# Run the binary program produced by `go install` -ENTRYPOINT ["./deveui-cli"] \ No newline at end of file +FROM scratch AS production +COPY --from=go-builder /app/deveui-cli /bin/ +ENTRYPOINT ["/bin/deveui-cli"] From 84a18401d7386c8deeb731c28d26d023e0bf0e45 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Wed, 17 May 2023 10:11:55 +0100 Subject: [PATCH 03/12] Add make file and docker compose file --- Makefile | 11 +++++++++++ docker-compose.yml | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 Makefile create mode 100644 docker-compose.yml diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3c1e92c --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +.PHONY: run +run: ## Run the app from source code + go run main.go + +.PHONY: docker +docker: ## Run the app in a container built from docker file + docker compose up app_docker + +.PHONY: local +local: ## Run the app in a container using source code + docker compose up app_local diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6e637df --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3.9" +services: + app_docker: + build: + context: . + dockerfile: dockerfile + volumes: + - ${GOPATH}/pkg/mod:/go/pkg/mod + + app_local: + image: golang:1.20 + volumes: + - .:/code + - ${GOPATH}/pkg/mod:/go/pkg/mod + working_dir: /code + command: go run main.go + + golangci: + image: golangci/golangci-lint:latest-alpine + volumes: + - .:/code + working_dir: /code + command: golangci-lint run -v From 544598ff9547a1c9d95474f36c42fe22a7de8d00 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Wed, 17 May 2023 16:13:58 +0100 Subject: [PATCH 04/12] Use envconfig in main --- go.mod | 2 +- go.sum | 4 ++-- main.go | 59 ++++++++++++++++----------------------------------------- 3 files changed, 19 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index a4ef303..1304d11 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,4 @@ module github.com/NickGowdy/deveui-cli go 1.20 -require github.com/joho/godotenv v1.5.1 // direct +require github.com/kelseyhightower/envconfig v1.4.0 diff --git a/go.sum b/go.sum index d61b19e..8642a1a 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,2 @@ -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= diff --git a/main.go b/main.go index c9bccb2..e6a5443 100644 --- a/main.go +++ b/main.go @@ -3,64 +3,37 @@ package main import ( "context" "net/http" - "os" - "strconv" "time" "github.com/NickGowdy/deveui-cli/client" "github.com/NickGowdy/deveui-cli/processor" - "github.com/joho/godotenv" + "github.com/kelseyhightower/envconfig" ) -/* -deveui-cli is a Go CLI program. -It is used for concurrently generating unique 16-character (hex) identifier called a DevEUI. -These are generated by the program and registered via an external (LoRaWAN) API. - -Usage: - - go run main.go (locally) - - go run deveui-cli (docker) +type config struct { + BaseURL string `envconfig:"BASE_URL" default:"http://europe-west1-machinemax-dev-d524.cloudfunctions.net"` + MaxConcurrent int `envconfig:"MAX_CONCURRENT" default:"10"` + CodeRegistrationLimit int `envconfig:"CODE_REGISTRATION_LIMIT" default:"100"` + Timeout time.Duration `envconfig:"TIMEOUT" default:"30s"` +} -Once this program starts, it will listen to syscall.SIGTERM, syscall.SIGINT via a channel. -This is to handle any unexpected terminations of the program and to resume processing DevEUIs. -*/ func main() { - if err := godotenv.Load(".env"); err != nil { - panic("error loading.env file") - } - - baseurl := os.Getenv("BASE_URL") - - maxConcurrentJobs, err := strconv.Atoi(os.Getenv("MAX_CONCURRENT_JOBS")) - if err != nil { - panic("error parsing MAX_CONCURRENT_JOBS to int") - } - - codeRegistrationLimit, err := strconv.Atoi(os.Getenv("CODE_REGISTRATION_LIMIT")) - if err != nil { - panic("error parsing CODE_REGISTRATION_LIMIT to int") - } - - timeout, err := strconv.Atoi(os.Getenv("TIMEOUT")) - if err != nil { - panic("error parsing TIMEOUT to int") + var cfg config + if err := envconfig.Process("", &cfg); err != nil { + panic("failed to process env vars: " + err.Error()) } - // setup client for requests httpClient := &http.Client{ - Timeout: time.Second * time.Duration(timeout), + Timeout: cfg.Timeout, } - loraWAN := client.NewLoraWAN(baseurl, httpClient) + loraWAN := client.NewLoraWAN(cfg.BaseURL, httpClient) - // setup processor to do work - processor := &processor.Processor{ - CodeRegistrationLimit: codeRegistrationLimit, - MaxConcurrentJobs: maxConcurrentJobs, + proc := &processor.Processor{ + CodeRegistrationLimit: cfg.CodeRegistrationLimit, + MaxConcurrentJobs: cfg.MaxConcurrent, LoraWAN: *loraWAN, } ctx, cancel := context.WithCancel(context.Background()) - processor.Start(ctx, cancel) + proc.Start(ctx, cancel) } From 7eeb24931833a0bcb936f7f825f006b3d24d8678 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Thu, 18 May 2023 08:18:29 +0100 Subject: [PATCH 05/12] Update lorawan client and tests --- client/lorawan.go | 72 ++++++++------- client/lorawan_test.go | 204 +++++++++++++++++++++++------------------ go.mod | 12 ++- go.sum | 19 ++++ main.go | 7 +- 5 files changed, 188 insertions(+), 126 deletions(-) diff --git a/client/lorawan.go b/client/lorawan.go index d832e17..2ae4fc9 100644 --- a/client/lorawan.go +++ b/client/lorawan.go @@ -4,64 +4,68 @@ import ( "bytes" "context" "encoding/json" - "errors" + "fmt" "net/http" + "net/url" + "time" "github.com/NickGowdy/deveui-cli/device" ) -// Client used to communicate to external services -type Client interface { - Do(*http.Request) (resp *http.Response, err error) -} - // LoraWAN used to communicate to LoRaWAN external system type LoraWAN struct { - baseURL string - client Client -} - -func NewLoraWAN(baseURL string, client Client) *LoraWAN { - return &LoraWAN{ - baseURL: baseURL, - client: client, - } + fullURL *url.URL + client *http.Client } const endpoint = "/sensor-onboarding-sample" // endpoint for saving DevEUI via LoRaWAN -// RegisterDevice registers new device using LoraWAN external service -func (l *LoraWAN) RegisterDevice(ctx context.Context) (*device.Device, error) { - device := device.NewDevice() - identifier := device.GetIdentifier() - b := new(bytes.Buffer) - reqBody := map[string]string{"Deveui": identifier} - - err := json.NewEncoder(b).Encode(&reqBody) +func NewLoraWAN(baseURL string, timeout time.Duration) (*LoraWAN, error) { + path, err := url.JoinPath(baseURL, endpoint) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to join url: %w", err) } - fullUrl := l.baseURL + endpoint - req, err := http.NewRequestWithContext(ctx, "POST", fullUrl, b) + fullURL, err := url.Parse(path) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse url: %w", err) } - resp, err := l.client.Do(req) + return &LoraWAN{ + fullURL: fullURL, + client: &http.Client{ + Timeout: timeout, + }, + }, nil +} + +// RegisterDevice registers a new device using LoraWAN external service +func (l *LoraWAN) RegisterDevice(ctx context.Context) (*device.Device, error) { + newDevice := device.NewDevice() + reqBody := map[string]string{"deveui": newDevice.GetIdentifier()} + + b := new(bytes.Buffer) + if err := json.NewEncoder(b).Encode(&reqBody); err != nil { + return nil, fmt.Errorf("failed to encode request body: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", l.fullURL.String(), b) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create request: %w", err) } + req.Header.Set("Content-Type", "application/json") + + resp, err := l.client.Do(req) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() - if resp.StatusCode == http.StatusOK { - return device, nil - } else { - return nil, errors.New(resp.Status) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return nil, fmt.Errorf("request failed: %s", resp.Status) } + + return newDevice, nil } diff --git a/client/lorawan_test.go b/client/lorawan_test.go index e10ddf3..b3a2eb4 100644 --- a/client/lorawan_test.go +++ b/client/lorawan_test.go @@ -1,95 +1,125 @@ package client import ( + "context" + "encoding/json" + "io" "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" + + "github.com/NickGowdy/deveui-cli/device" ) -type MockClient struct { - DoFunc func(*http.Request) (resp *http.Response, err error) +func TestLoraWAN_RegisterDevice_Request(t *testing.T) { + t.Parallel() + + const timeout = 10 * time.Second + + t.Run("server-side-checks", func(t *testing.T) { + t.Parallel() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, endpoint, r.URL.Path) + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + + data, err := io.ReadAll(r.Body) + defer r.Body.Close() + require.NoError(t, err) + var body struct { + Deveui string `json:"deveui"` + } + err = json.Unmarshal(data, &body) + require.NoError(t, err) + assert.NotEmpty(t, body.Deveui) + + w.WriteHeader(http.StatusCreated) + })) + defer ts.Close() + + client, err := NewLoraWAN(ts.URL, timeout) + require.NoError(t, err) + + _, err = client.RegisterDevice(context.Background()) + assert.NoError(t, err) + }) } -// func TestLorawanClientHappyPath(t *testing.T) { -// mockClient := &MockClient{ -// DoFunc: func(*http.Request) (resp *http.Response, err error) { -// return &http.Response{}, nil -// }, -// } - -// loraWAN := NewLoraWAN("www.example.com", mockClient) - -// b := new(bytes.Buffer) -// reqBody := map[string]string{"Deveui": "Abcde"} - -// _ = json.NewEncoder(b).Encode(&reqBody) - -// ctx, cancel := context.WithCancel(context.Background()) - -// if cancel == nil { -// t.Errorf("cancel should not be nil but is: %v", cancel) -// } - -// resp, err := loraWAN.DoPost(b, ctx) - -// if err != nil { -// t.Errorf("err should be nil but is: %s", err.Error()) -// } -// defer resp.Body.Close() - -// if resp.StatusCode != 200 { -// t.Errorf("resp should be nil but is: %d", resp.StatusCode) -// } - -// body, _ := io.ReadAll(resp.Body) -// val := string(body) - -// if strings.TrimSpace(val) != "true" { -// t.Errorf("body should equal true but is: %d", body) -// } -// } - -// func TestNewLoraWanClient(t *testing.T) { -// client := &http.Client{ -// Timeout: 30 * time.Second, -// } -// t.Parallel() -// type args struct { -// timeout time.Duration -// } -// tests := []struct { -// name string -// args args -// want *LoraWAN -// }{ -// { -// name: "create-new-lorawan-client", -// args: args{ -// timeout: 30, -// }, -// want: &LoraWAN{ -// baseURL: "https://www.example.com", -// client: client, -// }, -// }, -// } -// for _, tt := range tests { -// tt := tt // it is important to capture range variable -// t.Run(tt.name, func(t *testing.T) { -// t.Parallel() // this makes sure that all cases from the table here are executed in parallel -// if got := NewLoraWAN("https://www.example.com", client); !reflect.DeepEqual(got, tt.want) { -// t.Errorf("NewLoraWanClient() = %v, want %v", got, tt.want) -// } -// }) -// } -// } - -// func (m *MockClient) Do(*http.Request) (resp *http.Response, err error) { -// b := new(bytes.Buffer) -// reqBody := true - -// _ = json.NewEncoder(b).Encode(&reqBody) -// return &http.Response{ -// StatusCode: http.StatusOK, -// Body: io.NopCloser(b), -// Status: "200 OK"}, -// nil -// } +func TestLoraWAN_RegisterDevice(t *testing.T) { + t.Parallel() + type fields struct { + handler func(w http.ResponseWriter, r *http.Request) + } + tests := []struct { + name string + fields fields + want *device.Device + wantErr assert.ErrorAssertionFunc + }{ + { + name: "error-status-code", + fields: fields{ + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnprocessableEntity) + }, + }, + want: nil, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorContains(t, err, "request failed") + }, + }, + { + name: "ok-status-code", + fields: fields{ + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }, + }, + want: &device.Device{}, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.NoError(t, err) + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + r := chi.NewRouter() + r.Post(endpoint, tt.fields.handler) + ts := httptest.NewServer(r) + defer ts.Close() + + tsPath, err := url.JoinPath(ts.URL, endpoint) + require.NoError(t, err) + + tsURL, err := url.Parse(tsPath) + require.NoError(t, err) + + l := &LoraWAN{ + fullURL: tsURL, + client: ts.Client(), + } + got, err := l.RegisterDevice(context.Background()) + if !tt.wantErr(t, err, "RegisterDevice()") { + return + } + + // returned device has random field values, so we just check if what we got is not nil + if tt.want != nil { + assert.NotNil(t, got) + } else { + assert.Nil(t, got) + } + }) + } +} diff --git a/go.mod b/go.mod index 1304d11..8e1c629 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,14 @@ module github.com/NickGowdy/deveui-cli go 1.20 -require github.com/kelseyhightower/envconfig v1.4.0 +require ( + github.com/go-chi/chi/v5 v5.0.8 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/stretchr/testify v1.8.2 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index 8642a1a..ba79775 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,21 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= +github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index e6a5443..5c3242b 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "net/http" "time" "github.com/NickGowdy/deveui-cli/client" @@ -23,10 +22,10 @@ func main() { panic("failed to process env vars: " + err.Error()) } - httpClient := &http.Client{ - Timeout: cfg.Timeout, + loraWAN, err := client.NewLoraWAN(cfg.BaseURL, cfg.Timeout) + if err != nil { + panic("failed to create loraWAN client: " + err.Error()) } - loraWAN := client.NewLoraWAN(cfg.BaseURL, httpClient) proc := &processor.Processor{ CodeRegistrationLimit: cfg.CodeRegistrationLimit, From 2e1866bd543c79f8ee6c68bf1bda699ca0381810 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Thu, 18 May 2023 17:36:19 +0100 Subject: [PATCH 06/12] Improve RegisterDevice --- client/lorawan.go | 13 +++---- client/lorawan_test.go | 75 +++++++++++++++++++++++++++++++------ go.sum | 1 + processor/code_processor.go | 7 ++-- 4 files changed, 74 insertions(+), 22 deletions(-) diff --git a/client/lorawan.go b/client/lorawan.go index 2ae4fc9..d31441d 100644 --- a/client/lorawan.go +++ b/client/lorawan.go @@ -40,32 +40,31 @@ func NewLoraWAN(baseURL string, timeout time.Duration) (*LoraWAN, error) { } // RegisterDevice registers a new device using LoraWAN external service -func (l *LoraWAN) RegisterDevice(ctx context.Context) (*device.Device, error) { - newDevice := device.NewDevice() +func (l *LoraWAN) RegisterDevice(ctx context.Context, newDevice *device.Device) error { reqBody := map[string]string{"deveui": newDevice.GetIdentifier()} b := new(bytes.Buffer) if err := json.NewEncoder(b).Encode(&reqBody); err != nil { - return nil, fmt.Errorf("failed to encode request body: %w", err) + return fmt.Errorf("failed to encode request body: %w", err) } req, err := http.NewRequestWithContext(ctx, "POST", l.fullURL.String(), b) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", "application/json") resp, err := l.client.Do(req) if err != nil { - return nil, fmt.Errorf("failed to send request: %w", err) + return fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode > 299 { - return nil, fmt.Errorf("request failed: %s", resp.Status) + return fmt.Errorf("request failed: %s", resp.Status) } - return newDevice, nil + return nil } diff --git a/client/lorawan_test.go b/client/lorawan_test.go index b3a2eb4..2eee0d1 100644 --- a/client/lorawan_test.go +++ b/client/lorawan_test.go @@ -3,6 +3,7 @@ package client import ( "context" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -18,6 +19,63 @@ import ( "github.com/NickGowdy/deveui-cli/device" ) +func TestNewLoraWAN(t *testing.T) { + t.Parallel() + type args struct { + baseURL string + timeout time.Duration + } + tests := []struct { + name string + args args + want *LoraWAN + wantErr assert.ErrorAssertionFunc + }{ + { + name: "invalid-base-url", + args: args{ + baseURL: ":invalid", + }, + want: nil, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorContains(t, err, "failed to join url") + }, + }, + { + name: "ok", + args: args{ + baseURL: "base", + timeout: 1 * time.Second, + }, + want: &LoraWAN{ + fullURL: func() *url.URL { + u, err := url.Parse(fmt.Sprintf("%s%s", "base", endpoint)) + require.NoError(t, err) + + return u + }(), + client: &http.Client{ + Timeout: 1 * time.Second, + }, + }, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.NoError(t, err) + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := NewLoraWAN(tt.args.baseURL, tt.args.timeout) + if !tt.wantErr(t, err, fmt.Sprintf("NewLoraWAN(%v, %v)", tt.args.baseURL, tt.args.timeout)) { + return + } + assert.EqualValues(t, tt.want, got) + }) + } +} + func TestLoraWAN_RegisterDevice_Request(t *testing.T) { t.Parallel() @@ -26,6 +84,8 @@ func TestLoraWAN_RegisterDevice_Request(t *testing.T) { t.Run("server-side-checks", func(t *testing.T) { t.Parallel() + newDevice := device.NewDevice() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, http.MethodPost, r.Method) assert.Equal(t, endpoint, r.URL.Path) @@ -39,7 +99,7 @@ func TestLoraWAN_RegisterDevice_Request(t *testing.T) { } err = json.Unmarshal(data, &body) require.NoError(t, err) - assert.NotEmpty(t, body.Deveui) + assert.Equal(t, newDevice.GetIdentifier(), body.Deveui) w.WriteHeader(http.StatusCreated) })) @@ -48,7 +108,7 @@ func TestLoraWAN_RegisterDevice_Request(t *testing.T) { client, err := NewLoraWAN(ts.URL, timeout) require.NoError(t, err) - _, err = client.RegisterDevice(context.Background()) + err = client.RegisterDevice(context.Background(), newDevice) assert.NoError(t, err) }) } @@ -109,17 +169,8 @@ func TestLoraWAN_RegisterDevice(t *testing.T) { fullURL: tsURL, client: ts.Client(), } - got, err := l.RegisterDevice(context.Background()) - if !tt.wantErr(t, err, "RegisterDevice()") { - return - } - // returned device has random field values, so we just check if what we got is not nil - if tt.want != nil { - assert.NotNil(t, got) - } else { - assert.Nil(t, got) - } + tt.wantErr(t, l.RegisterDevice(context.Background(), &device.Device{}), "RegisterDevice()") }) } } diff --git a/go.sum b/go.sum index ba79775..34caa62 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= diff --git a/processor/code_processor.go b/processor/code_processor.go index 143c881..354598d 100644 --- a/processor/code_processor.go +++ b/processor/code_processor.go @@ -5,6 +5,7 @@ import ( "log" "github.com/NickGowdy/deveui-cli/client" + "github.com/NickGowdy/deveui-cli/device" ) type Processor struct { @@ -18,11 +19,11 @@ func (p *Processor) Start(ctx context.Context, cancel context.CancelFunc) { count := 0 for count < p.CodeRegistrationLimit { - device, err := p.LoraWAN.RegisterDevice(ctx) - if err != nil { + newDevice := device.NewDevice() + if err := p.LoraWAN.RegisterDevice(ctx, newDevice); err != nil { log.Print(err) } else { - device.Print() + newDevice.Print() count++ } } From af6213fb5a74f323eef43732216d7b2a2b984c8d Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Fri, 19 May 2023 08:07:32 +0100 Subject: [PATCH 07/12] Update device code --- client/lorawan_test.go | 3 +- device/device.go | 34 ++++++-------- device/device_test.go | 92 ++++++------------------------------- processor/code_processor.go | 20 +++++--- 4 files changed, 42 insertions(+), 107 deletions(-) diff --git a/client/lorawan_test.go b/client/lorawan_test.go index 2eee0d1..3256e12 100644 --- a/client/lorawan_test.go +++ b/client/lorawan_test.go @@ -84,7 +84,8 @@ func TestLoraWAN_RegisterDevice_Request(t *testing.T) { t.Run("server-side-checks", func(t *testing.T) { t.Parallel() - newDevice := device.NewDevice() + newDevice, err := device.NewDevice() + require.NoError(t, err) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, http.MethodPost, r.Method) diff --git a/device/device.go b/device/device.go index 6e94089..2a44dd3 100644 --- a/device/device.go +++ b/device/device.go @@ -3,13 +3,12 @@ package device import ( "crypto/rand" "fmt" - "log" "math/big" ) const ( - AllowedChars = "ABCDEF0123456789" // accepted chars used to make up DevEUI - DevEuiLength = 16 // valid DevEUI is string of length 16 + allowedChars = "ABCDEF0123456789" // accepted chars used to make up DevEUI + devEuiLength = 16 // valid DevEUI is string of length 16 ) type Device struct { @@ -18,20 +17,17 @@ type Device struct { } // NewDevice Build a new device with DevEUI identifier and code values. -// -// # Example -// -// 1CEB0080F074F750 4F750 -func NewDevice() *Device { +// Example: 1CEB0080F074F750 4F750 +func NewDevice() (*Device, error) { hex, err := generateHexString() if err != nil { - log.Fatal(err) + return nil, fmt.Errorf("failed to generate DevEUI: %w", err) } return &Device{ identifier: hex, code: hex[len(hex)-5:], - } + }, nil } func (d Device) GetIdentifier() string { @@ -42,24 +38,22 @@ func (d Device) GetCode() string { return d.code } -func (d Device) Print() { - fmt.Printf("device has identifier: %s and code: %s\n", d.identifier, d.code) +// String returns a string representation of the device (see Stringer interface at https://go.dev/tour/methods/17) +func (d Device) String() string { + return fmt.Sprintf("device has identifier: %s and code: %s", d.identifier, d.code) } // Generate valid DevEUI identifier value. -// -// # Example -// -// 1CEB0080F074F750 +// Example: 1CEB0080F074F750 func generateHexString() (string, error) { - max := big.NewInt(int64(len(AllowedChars))) - b := make([]byte, DevEuiLength) + max := big.NewInt(int64(len(allowedChars))) + b := make([]byte, devEuiLength) for i := range b { n, err := rand.Int(rand.Reader, max) if err != nil { - return "", err + return "", fmt.Errorf("failed to generate random int: %w", err) } - b[i] = AllowedChars[n.Int64()] + b[i] = allowedChars[n.Int64()] } return string(b), nil } diff --git a/device/device_test.go b/device/device_test.go index 46fa331..7e4cc34 100644 --- a/device/device_test.go +++ b/device/device_test.go @@ -1,90 +1,24 @@ package device import ( + "github.com/stretchr/testify/assert" + "regexp" "testing" -) - -func TestCanGenerateValidCode(t *testing.T) { - allowedChars := []string{"A", "B", "C", "D", "E", "F", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"} - device := NewDevice() - - if device == nil { - t.Errorf("deivce should not be nil, but is %s", device) - } - - identifier := device.GetIdentifier() - code := device.GetCode() - - if len(code) != 5 { - t.Errorf("code should be 5 characters long, but is %d", len(code)) - } - - if identifier[len(identifier)-5:] != code { - t.Errorf("code should be last 5 characters of identifier, but is %s", code) - } - hasChar := false - for _, char := range allowedChars { - if char == string(code[0]) { - hasChar = true - } else if hasChar { - break - } - } - - if hasChar == false { - t.Errorf("first char should be A, B, C, D, E, F, 0, 1, 2, 3, 4, 5, 6, 7, 8 or 9: but is: %s", string(code[0])) - } - - hasChar = false - for _, char := range allowedChars { - if char == string(code[1]) { - hasChar = true - } else if hasChar { - break - } - } - - if hasChar == false { - t.Errorf("second char should be A, B, C, D, E, F, 0, 1, 2, 3, 4, 5, 6, 7, 8 or 9: but is: %s", string(code[1])) - } - - hasChar = false - for _, char := range allowedChars { - if char == string(code[2]) { - hasChar = true - } else if hasChar { - break - } - } + "github.com/stretchr/testify/require" +) - if hasChar == false { - t.Errorf("third char should be A, B, C, D, E, F, 0, 1, 2, 3, 4, 5, 6, 7, 8 or 9: but is: %s", string(code[2])) - } +func TestNewDevice(t *testing.T) { + t.Parallel() - hasChar = false - for _, char := range allowedChars { - if char == string(code[3]) { - hasChar = true - } else if hasChar { - break - } - } + got, err := NewDevice() + require.NoError(t, err) - if hasChar == false { - t.Errorf("fourth char should be A, B, C, D, E, F, 0, 1, 2, 3, 4, 5, 6, 7, 8 or 9: but is: %s", string(code[3])) - } + // regex can be tested and explained here: https://regex101.com/ - hasChar = false - for _, char := range allowedChars { - if char == string(code[4]) { - hasChar = true - } else if hasChar { - break - } - } + reId := regexp.MustCompile("^[0-9A-F]{16}$") + assert.True(t, reId.MatchString(got.GetIdentifier())) - if hasChar == false { - t.Errorf("fifth char should be A, B, C, D, E, F, 0, 1, 2, 3, 4, 5, 6, 7, 8 or 9: but is: %s", string(code[4])) - } + reCode := regexp.MustCompile("^[0-9A-F]{5}$") + assert.True(t, reCode.MatchString(got.GetCode())) } diff --git a/processor/code_processor.go b/processor/code_processor.go index 354598d..8e90f03 100644 --- a/processor/code_processor.go +++ b/processor/code_processor.go @@ -18,14 +18,20 @@ func (p *Processor) Start(ctx context.Context, cancel context.CancelFunc) { // workCh := make(chan struct{}) count := 0 - for count < p.CodeRegistrationLimit { - newDevice := device.NewDevice() - if err := p.LoraWAN.RegisterDevice(ctx, newDevice); err != nil { - log.Print(err) - } else { - newDevice.Print() - count++ + for count < p.codeRegistrationLimit { + newDevice, err := device.NewDevice() + if err != nil { + log.Printf("failed to create new device: %v\n", err) + continue } + + if err = p.client.RegisterDevice(ctx, newDevice); err != nil { + log.Printf("failed to register device: %v\n", err) + continue + } + + log.Println(newDevice) + count++ } // go func(ctx context.Context) { From a4eff38a0ce247757b7f7c67ba66fc75cc2ca4ab Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Fri, 19 May 2023 08:07:43 +0100 Subject: [PATCH 08/12] Update main --- main.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index 5c3242b..5655916 100644 --- a/main.go +++ b/main.go @@ -17,22 +17,28 @@ type config struct { } func main() { + cfg := mustReadConfig() + loraWAN := mustCreateClient(cfg.BaseURL, cfg.Timeout) + proc := processor.New(cfg.CodeRegistrationLimit, cfg.MaxConcurrent, loraWAN) + + ctx, cancel := context.WithCancel(context.Background()) + proc.Start(ctx, cancel) +} + +func mustReadConfig() config { var cfg config if err := envconfig.Process("", &cfg); err != nil { panic("failed to process env vars: " + err.Error()) } - loraWAN, err := client.NewLoraWAN(cfg.BaseURL, cfg.Timeout) + return cfg +} + +func mustCreateClient(baseURL string, timeout time.Duration) *client.LoraWAN { + loraWAN, err := client.NewLoraWAN(baseURL, timeout) if err != nil { panic("failed to create loraWAN client: " + err.Error()) } - proc := &processor.Processor{ - CodeRegistrationLimit: cfg.CodeRegistrationLimit, - MaxConcurrentJobs: cfg.MaxConcurrent, - LoraWAN: *loraWAN, - } - - ctx, cancel := context.WithCancel(context.Background()) - proc.Start(ctx, cancel) + return loraWAN } From d85e17d6fffd16e511fa133c2c3a38afdad2eae6 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Fri, 19 May 2023 08:08:12 +0100 Subject: [PATCH 09/12] Make Processor fields private --- processor/code_processor.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/processor/code_processor.go b/processor/code_processor.go index 8e90f03..75c5ae4 100644 --- a/processor/code_processor.go +++ b/processor/code_processor.go @@ -4,14 +4,28 @@ import ( "context" "log" - "github.com/NickGowdy/deveui-cli/client" "github.com/NickGowdy/deveui-cli/device" ) -type Processor struct { - CodeRegistrationLimit int - MaxConcurrentJobs int - LoraWAN client.LoraWAN +type ( + // Client is the client interface the processor expects + Client interface { + RegisterDevice(ctx context.Context, newDevice *device.Device) error + } + + Processor struct { + codeRegistrationLimit int + maxConcurrentJobs int + client Client + } +) + +func New(codeRegistrationLimit int, maxConcurrentJobs int, client Client) *Processor { + return &Processor{ + codeRegistrationLimit: codeRegistrationLimit, + maxConcurrentJobs: maxConcurrentJobs, + client: client, + } } func (p *Processor) Start(ctx context.Context, cancel context.CancelFunc) { From fea693468b2693007627d577425a2cc9bac273e4 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Fri, 26 May 2023 08:52:37 +0100 Subject: [PATCH 10/12] Update device print text --- device/device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/device/device.go b/device/device.go index 2a44dd3..d7a0d6c 100644 --- a/device/device.go +++ b/device/device.go @@ -40,7 +40,7 @@ func (d Device) GetCode() string { // String returns a string representation of the device (see Stringer interface at https://go.dev/tour/methods/17) func (d Device) String() string { - return fmt.Sprintf("device has identifier: %s and code: %s", d.identifier, d.code) + return fmt.Sprintf("{id: %s, code: %s}", d.identifier, d.code) } // Generate valid DevEUI identifier value. From 0db88d0aded472edbfcfc99d10d7c73447c769b8 Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Fri, 26 May 2023 08:53:58 +0100 Subject: [PATCH 11/12] Implement processor logic --- processor/code_processor.go | 118 +++++++++++++++++++++++------------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/processor/code_processor.go b/processor/code_processor.go index 75c5ae4..6740e67 100644 --- a/processor/code_processor.go +++ b/processor/code_processor.go @@ -18,6 +18,9 @@ type ( maxConcurrentJobs int client Client } + + // T is a short alias for a blank struct, which has zero size + T struct{} ) func New(codeRegistrationLimit int, maxConcurrentJobs int, client Client) *Processor { @@ -28,52 +31,81 @@ func New(codeRegistrationLimit int, maxConcurrentJobs int, client Client) *Proce } } +func (p *Processor) printDevice(done chan<- T, ch <-chan *device.Device) { + // count is not under race condition here + count := 1 + for dev := range ch { + log.Printf("registered device %d/%d: %s\n", count, p.codeRegistrationLimit, dev.String()) + count++ + } + done <- T{} +} + func (p *Processor) Start(ctx context.Context, cancel context.CancelFunc) { - // workCh := make(chan struct{}) - count := 0 - - for count < p.codeRegistrationLimit { - newDevice, err := device.NewDevice() - if err != nil { - log.Printf("failed to create new device: %v\n", err) - continue - } + done := make(chan T) + devicePrinter := make(chan *device.Device) + go p.printDevice(done, devicePrinter) - if err = p.client.RegisterDevice(ctx, newDevice); err != nil { - log.Printf("failed to register device: %v\n", err) - continue - } + workers := make(chan T, p.maxConcurrentJobs) + for i := 0; i < p.maxConcurrentJobs; i++ { + workers <- T{} + } - log.Println(newDevice) - count++ + // NOTE: use a buffered channel to track successfully completed jobs + // First, fill up the channel to max capacity, then during the loop take + // one item until the channel is closed. The channel is closed when it becomes + // empty. + leftToDo := make(chan T, p.codeRegistrationLimit) + for i := 0; i < p.codeRegistrationLimit; i++ { + leftToDo <- T{} } - // go func(ctx context.Context) { - // for { - // p.doWork(ctx, workCh) - // } - // }(ctx) - - // for { - // select { - // case <-ctx.Done(): - // return - // case <-workCh: - // count++ - // if count == p.CodeRegistrationLimit { - // cancel() - // fmt.Printf("work complete \n") - // } - // } - // } -} + // This will block when there are only in-flight jobs running and we wait for their outcome. In other words, + // we don't start any new jobs until we have in-flight jobs and they have a chance to satisfy the requested + // number of registrations. + // When the channel is closed, we know that all jobs have been completed. + for range leftToDo { + select { + case <-ctx.Done(): + // graceful shutdown + log.Printf("processor shutting down") -// func (cp *Processor) doWork(ctx context.Context, workCh chan<- struct{}) { -// device, err := cp.LoraWAN.RegisterDevice(ctx) -// if err != nil { -// return -// } else { -// device.Print() -// workCh <- struct{}{} -// } -// } + return + default: + if _, ok := <-workers; ok { + go func() { + defer func() { + workers <- T{} + + if len(workers) == cap(workers) && len(leftToDo) == 0 { + close(leftToDo) + close(workers) + close(devicePrinter) + } + }() + + newDevice, err := device.NewDevice() + if err != nil { + log.Printf("failed to create new device: %v\n", err) + leftToDo <- T{} + + return + } + + if err = p.client.RegisterDevice(ctx, newDevice); err != nil { + log.Printf("failed to register device: %v\n", err) + leftToDo <- T{} + + return + } + + devicePrinter <- newDevice + }() + } + } + } + + <-done + log.Println("all jobs have been completed") + cancel() +} From 93dc4b34f676e42cf0b5ebe1d6258294c73e6e2e Mon Sep 17 00:00:00 2001 From: Gabor Szabad Date: Fri, 26 May 2023 08:54:11 +0100 Subject: [PATCH 12/12] Add graceful shutdown --- device/device_test.go | 2 +- main.go | 46 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/device/device_test.go b/device/device_test.go index 7e4cc34..780e8e2 100644 --- a/device/device_test.go +++ b/device/device_test.go @@ -1,10 +1,10 @@ package device import ( - "github.com/stretchr/testify/assert" "regexp" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/main.go b/main.go index 5655916..4eb3103 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,10 @@ package main import ( "context" + "log" + "os" + "os/signal" + "syscall" "time" "github.com/NickGowdy/deveui-cli/client" @@ -22,7 +26,10 @@ func main() { proc := processor.New(cfg.CodeRegistrationLimit, cfg.MaxConcurrent, loraWAN) ctx, cancel := context.WithCancel(context.Background()) - proc.Start(ctx, cancel) + + go proc.Start(ctx, cancel) + + handleGracefulShutdown(ctx, cancel) } func mustReadConfig() config { @@ -42,3 +49,40 @@ func mustCreateClient(baseURL string, timeout time.Duration) *client.LoraWAN { return loraWAN } + +func handleGracefulShutdown(ctx context.Context, cancel context.CancelFunc) { + const ( + shutdownTimeout = 5 * time.Second + + shutdownTimeoutExpired = 1 + forcedExit = 2 + ) + + // create interrupt signal listener, use the app's context + ctx, _ = signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + + // block and listen for the interrupt signal + <-ctx.Done() + + log.Println("shutting down") + + // let the ctx chain know that the app is terminating + cancel() + + go func() { + // restart listening for the force exist signal, any new context will do here + // e.g. press Ctrl+C again to force + ctx2, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + + select { + case <-time.After(shutdownTimeout): + log.Println("shutdown timeout expired") + + os.Exit(shutdownTimeoutExpired) + case <-ctx2.Done(): + log.Println("forced exit") + + os.Exit(forcedExit) + } + }() +}