Creating a Terraform Provider for LoRaServer

Hi all,

We’re using terraform.io to provision all of our infrastructure and monitoring, and we thought it would be really cool to create a terraform provider to configure LoRaServer at the same time as we deploy it.

We’ve created a git repository for it, and we’re willing to shepherd it through the process, except we can’t find any documentation on how to use the go client listed at https://www.loraserver.io/lora-app-server/integrate/grpc/

Is this something others would be willing to work on with us?

This is a simple example how you could import a list of devices from an excel file. It should give you some information how you can use the gRPC interface:

package main

import (
	"context"
	"crypto/tls"
	"flag"
	"log"

	"github.com/pkg/errors"
	"github.com/tealeg/xlsx"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"

	"github.com/brocaar/lora-app-server/api"
)

// JWTCredentials provides JWT credentials for gRPC
type JWTCredentials struct {
	token string
}

// GetRequestMetadata returns the meta-data for a request.
func (j *JWTCredentials) GetRequestMetadata(ctx context.Context, url ...string) (map[string]string, error) {
	return map[string]string{
		"authorization": j.token,
	}, nil
}

// RequireTransportSecurity ...
func (j *JWTCredentials) RequireTransportSecurity() bool {
	return false
}

// SetToken sets the JWT token.
func (j *JWTCredentials) SetToken(token string) {
	j.token = token
}

// DeviceImportRecord defines a record for a device to import.
type DeviceImportRecord struct {
	DevEUI          string
	ApplicationID   int64
	DeviceProfileID string
	Name            string
	Description     string
	NetworkKey      string
	ApplicationKey  string
}

var (
	username       string
	password       string
	file           string
	apiHost        string
	apiInsecure    bool
	jwtCredentials *JWTCredentials
)

func init() {
	jwtCredentials = &JWTCredentials{}

	flag.StringVar(&username, "username", "admin", "LoRa App Server username")
	flag.StringVar(&password, "password", "admin", "LoRa App Server password")
	flag.StringVar(&file, "file", "", "Path to Excel file")
	flag.StringVar(&apiHost, "api", "localhost:8080", "hostname:port to LoRa App Server API")
	flag.BoolVar(&apiInsecure, "api-insecure", false, "LoRa App Server API does not use TLS")
	flag.Parse()
}

func getGRPCConn() (*grpc.ClientConn, error) {
	dialOpts := []grpc.DialOption{
		grpc.WithBlock(),
		grpc.WithPerRPCCredentials(jwtCredentials),
	}

	if apiInsecure {
		log.Println("using insecure api")
		dialOpts = append(dialOpts, grpc.WithInsecure())
	} else {
		dialOpts = append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
			InsecureSkipVerify: true,
		})))
	}

	conn, err := grpc.Dial(apiHost, dialOpts...)
	if err != nil {
		return nil, errors.Wrap(err, "grpc dial error")
	}

	return conn, nil
}

func login(conn *grpc.ClientConn) error {
	internalClient := api.NewInternalServiceClient(conn)

	resp, err := internalClient.Login(context.Background(), &api.LoginRequest{
		Username: username,
		Password: password,
	})
	if err != nil {
		return errors.Wrap(err, "login error")
	}

	jwtCredentials.SetToken(resp.Jwt)

	return nil
}

func getDeviceImportList() ([]DeviceImportRecord, error) {
	xlFile, err := xlsx.OpenFile(file)
	if err != nil {
		return nil, errors.Wrap(err, "open excel file error")
	}

	var out []DeviceImportRecord

	for _, sheet := range xlFile.Sheets {
		for i, row := range sheet.Rows {
			if i == 0 {
				continue
			}

			if len(row.Cells) != 7 {
				log.Fatalf("expected exactly 7 columns (row %d)", i+1)
			}

			devEUI := row.Cells[0].String()
			applicationID, err := row.Cells[1].Int64()
			if err != nil {
				log.Fatalf("application id parse error (row %d): %s", i+1, err)
			}
			deviceProfileID := row.Cells[2].String()
			name := row.Cells[3].String()
			description := row.Cells[4].String()
			networkKey := row.Cells[5].String()
			applicationKey := row.Cells[6].String()

			out = append(out, DeviceImportRecord{
				DevEUI:          devEUI,
				ApplicationID:   applicationID,
				DeviceProfileID: deviceProfileID,
				Name:            name,
				Description:     description,
				NetworkKey:      networkKey,
				ApplicationKey:  applicationKey,
			})
		}
	}

	return out, nil
}

func importDevices(conn *grpc.ClientConn, devices []DeviceImportRecord) error {
	deviceClient := api.NewDeviceServiceClient(conn)

	for i, dev := range devices {
		d := api.Device{
			DevEui:          dev.DevEUI,
			Name:            dev.Name,
			ApplicationId:   dev.ApplicationID,
			Description:     dev.Description,
			DeviceProfileId: dev.DeviceProfileID,
		}

		dk := api.DeviceKeys{
			DevEui: dev.DevEUI,
			NwkKey: dev.NetworkKey,
			AppKey: dev.ApplicationKey,
		}

		_, err := deviceClient.Create(context.Background(), &api.CreateDeviceRequest{
			Device: &d,
		})
		if err != nil {
			if grpc.Code(err) == codes.AlreadyExists {
				log.Printf("device %s already exists (row %d)", d.DevEui, i+2)
				continue
			}
			log.Fatalf("import error (device %s row %d): %s", d.DevEui, i+2, err)
		}

		_, err = deviceClient.CreateKeys(context.Background(), &api.CreateDeviceKeysRequest{
			DeviceKeys: &dk,
		})
		if err != nil {
			if grpc.Code(err) == codes.AlreadyExists {
				log.Printf("device-keys for device %s already exists (row %d)", d.DevEui, i+2)
				continue
			}
			log.Fatalf("import error (device %s) (row %d): %s", d.DevEui, i+2, err)
		}
	}

	return nil
}

func main() {
	conn, err := getGRPCConn()
	if err != nil {
		log.Fatal("error connecting to api", err)
	}

	if err := login(conn); err != nil {
		log.Fatal("login error", err)
	}

	rows, err := getDeviceImportList()
	if err != nil {
		log.Fatal("get device import records error", err)
	}

	if err := importDevices(conn, rows); err != nil {
		log.Fatal("import error", err)
	}
}
4 Likes

Thanks Brocaar, I’ll see if I can get somewhere with this.

I WAS trying to get this to work and HAD some questions:

  1. Where in the Chirpstack dashboard can I find the ApplicationID (in64) field for the device XLSX?
    Answer: integrations/mqtt page says to view a device and then look in the address bar (for me that’s “http://10.1.6.90:8080/#/organizations/1/applications/1/devices/4491600000f7d7bb”). So, my ApplicationID is just “1”?
    [Yes, this is correct.] [edit] Since I managed to accidentally delete my application (see below), that’s now “3”.

  2. In the DeviceImportRecord, what is/how do I determine the NetworkKey (string)? “NetworkKey” links to just this entry on forums.chirpstack.io (useless to me).
    Answer: I just used Device Key that all of my devices come with IN BOTH FIELDS (NetworkKey & ApplicationKey). Devices join fine.

  3. I also ‘guessed’ DeviceProfileID wrong - it must be the UUID of “Device-profiles/rb_decoded”. So, in Chripstack, go to a particular device profile and then look at address line. For me that was “http://10.1.6.90:8080/#/organizations/1/device-profiles/34c91614-d656-42f6-b3bf-edad433b90de”. The DeviceProfileID is the last string, but must remove all the dashes.

  4. Every time I ran the script, it exited with an error BUT it still added one sensor - it was seeing a space in first position of every ApplicationKey field but still adding device, sans AppKey. It should ignore leading & trailing spaces :slight_smile:

    2021/08/06 18:50:07 WARNING: proto: file “common.proto” is already registered
    A future release will panic on registration conflicts. See:
    https://developers.google.com/protocol-buffers/docs/reference/go/faq#namespace-conflict

    2021/08/06 18:50:07 using insecure api
    2021/08/06 18:50:07 device CCC0790000EE5595 already exists (row 2)
    2021/08/06 18:50:07 import error (device CCC0790000EE4DCC) (row 3): rpc error: code = InvalidArgument desc = encoding/hex: invalid byte: U+0020 ’ '
    exit status 1

In lieu of XLSX file, below is line for one device that I’m using:
DevEUI (str) = CCC0790000EE5595
ApplicationID (in64) = 1
DeviceProfileID (str) = 34c91614d65642f6b3bfedad433b90de
Name (str) = 301-C1
Description (str) = External_RTD_-40to100C
NetworkKey (str) = 00000000000000000000000000000000
ApplicationKey (str) = BBA84EB3883E1142996C9FA474D0E7C8

Too bad there is no column to specify value for “Disable frame-counter validation” as all of my devices have this turned off (ON by default). So all devices still require a manual edit :frowning:

It’s working now.

Thank you.

P.S. Thank you for this great software. I’m adding sensors in an older facility and not having to run conduit is a lifesaver.

So, I was happily deleting each device in Chirpstack and then I must have hit the DELETE for application, and lost 2 days of work. Drat.

Too bad in the interface, the DELETE for deleting devices is in exactly the same location as the DELETE for the application.

[edit]
BUT, since this is working now, I was able to update XLSX with ALL my devices from master spreadsheet and it’s all working again. Wow.

One thing that The Things Network V2 interface got right and that should be ‘copied’ to ChirpStack is the requirement to type in or copy Application ID when deleting an application. Here’s how that looks & works:
Delete_ApplicationID

It’s acceptable for devices to be deleted with with just a pop-up confirmation (but, really, there should be a way to select a list of devices and delete them in one operation), but accidentally deleting an Application can lead an user to tears.

(I’m an ‘army of 1’ so can’t take on more projects or I’d do it).