Mqtt encode/encrypt uplink data?

Hi everyone,

I have looked over different post here to find what i’m seeing on my mosquitto client when i subscribe to the mqqt broker, but i dont understand ! nothing at all…
could you help me?
i’m on a raspbian shell:

If i do: “mosquitto_sub -t “application/3/#” -v”
i get :

"application/3/device/60c5a8fffe783fc7/rx
                                          sondehumtempnode01`Ũ��x?�*Z
ܦ2��6\�
       �������N
                �������N(���������1������#@8@Zz���Ă����7�AԴ��v�zѢ�2�鄞     }
                                                                                         4/58HOZmh~s%�g�Hb�{"DecodeDataHex":"0802016d07687e067325d6026700a004020248","DecodeDataObj":{"battery":"3.65V","environment":{"barometer":"968.60hPa","gasResistance":"5.84KΩ","humidity":"63.0% RH","temperature":"16.00°C"}}}"

Ok the decoder give both decode data and the json data . but what are the data above?

The question is : in the mqtt subscribe, Have i an encrypted payload by the AES128 application key or a BASE64 encoded payload?
I tryed to decode the special character with base64 function but nothing clear…

To resume, where i could look for understand what is encrypted (AES128) , what is encoded(BASE64)?

if I would like to get the output events of the application server without codec transformation, How i could do that?

futhermore I try this “empty codec” on the webui application-server interface but i get only errors:

function Decode(fPort, bytes, variables) {
  return {};
}

many thanks for your feedback.

OK… simple !
marshaler have been move to protobuff…

If i replace to Json all looks betters!

Hi everyone,

After modify the marshaler i have thinked that all looks good to get the mqtt message but today i try to integrate it in a application and … i constat that i have not a readable json message.
look at the end inside objectJSON
specialy concerning the “\”

    {

	"applicationID":"3",

	"applicationName":"sondehumtemp",

	"deviceName":"node01",

	"devEUI":"YMWo//kkk8c=",

	"rxInfo":

		[

			{

				"gatewayID":"3KYy//4XXXXX",
				"timeSinceGPSEpoch":null,
				"rssi":-67,
				"crcStatus":"CRC_OK"
			}
		],
	"txInfo":
			{
			"frequency":867900000,
			"modulation":"LORA",
			"loRaModulationInfo":
				{
				"bandwidth":125,
				"spreadingFactor":12,
				"codeRate":"4/5",
				"polarizationInversion":false
				}
			},
	"adr":true,
	"fCnt":529,
	"fPort":8,
	"data":"CAIBbgdoXQZzJXcCZwDNBAIC4g==",
	"objectJSON":"
			{
			\"DecodeDataHex\":\"0802016e07685d06732577026700cd040202e2\",
			\"DecodeDataObj\":
				{
				\"battery\":\"3.66V\",
				\"environment\":
						{
						\"barometer\":\"959.10hPa\",
						\"gasResistance\":\"7.38KΩ\",
						\"humidity\":\"46.5% RH\",
						\"temperature\":\"20.50°C\"
						}
				}
			}",		
			"tags":{}
}

Why i get this backslash symbol “\” ?? as it announce a special charatere…
i suppose there is a mistake in my codec…
but i dont see why this backslash is added…
rak72xx payload codec from https://github.com/RAKWireless/RUI_LoRa_node_payload_decoder/blob/master/RUISensorDataDecoder_for_ChirpStack.js

// chirpstack application function to decode uplink data.
// Decode decodes an array of bytes into an object.
//  - fPort contains the LoRaWAN fPort number
//  - bytes is an array of bytes, e.g. [225, 230, 255, 0]
// The function must return an object
// for RAK, return {
//                     "DecodeDataHex": {} // RAK5205 sensor data in Hex format
//                     "DecodeDataObj": {} // RAK5205 sensor data object.
//                 }
// The function prototype cannot be modified.
function Decode(fPort, bytes) {
  var decoded = {"DecodeDataHex": {}, "DecodeDataObj": {}};
  var hexString=bin2HexStr(bytes);
  decoded.DecodeDataHex = hexString;
  decoded.DecodeDataObj = rakSensorDataDecode(hexString);

  return decoded;
}

// convert array of bytes to hex string.
// e.g: 0188053797109D5900DC140802017A0768580673256D0267011D040214AF0371FFFFFFDDFC2E
function bin2HexStr(bytesArr) {
  var str = "";
  for(var i=0; i<bytesArr.length; i++) {
    var tmp = (bytesArr[i] & 0xff).toString(16);
    if(tmp.length == 1) {
      tmp = "0" + tmp;
    }
    str += tmp;
  }
  return str;
}

// convert string to short integer
function parseShort(str, base) {
  var n = parseInt(str, base);
  return (n << 16) >> 16;
}

// convert string to triple bytes integer
function parseTriple(str, base) {
  var n = parseInt(str, base);
  return (n << 8) >> 8;
}

// decode Hex sensor string data to object
function rakSensorDataDecode(hexStr) {
  var str = hexStr;
  var myObj = {};
  var environment = {};
  var magnetometer = {};

  while (str.length > 4) {
    var flag = parseInt(str.substring(0, 4), 16);
    switch (flag) {
      case 0x0768:// Humidity
        environment.humidity = ((parseShort(str.substring(4, 6), 16) * 0.01 / 2) * 100).toFixed(1) + '% RH';
        str = str.substring(6);
        break;
      case 0x0673:// Atmospheric pressure
        environment.barometer = (parseShort(str.substring(4, 8), 16) * 0.1).toFixed(2) + "hPa";
        str = str.substring(8);
        break;
      case 0x0267:// Temperature
        environment.temperature = (parseShort(str.substring(4, 8), 16) * 0.1).toFixed(2) + "°C";
        str = str.substring(8);
        break;
      case 0x0188:// GPS
        var gps = {};
        gps.latitude = (parseTriple(str.substring(4, 10), 16) * 0.0001).toFixed(4) + "°";
        gps.longitude = (parseTriple(str.substring(10, 16), 16) * 0.0001).toFixed(4) + "°";
        gps.altitude = (parseTriple(str.substring(16, 22), 16) * 0.01).toFixed(1) + "m";
        myObj.gps = gps;
        str = str.substring(22);
        break;
      case 0x0371:// Triaxial acceleration
        var acceleration = {};
        acceleration.x = (parseShort(str.substring(4, 8), 16) * 0.001).toFixed(3) + "g";
        acceleration.y = (parseShort(str.substring(8, 12), 16) * 0.001).toFixed(3) + "g";
        acceleration.z = (parseShort(str.substring(12, 16), 16) * 0.001).toFixed(3) + "g";
        myObj.acceleration = acceleration;
        str = str.substring(16);
        break;
      case 0x0402:// air resistance
        environment.gasResistance = (parseShort(str.substring(4, 8), 16) * 0.01).toFixed(2)  + "KΩ";
        str = str.substring(8);
        break;
      case 0x0802:// Battery Voltage
        myObj.battery = (parseShort(str.substring(4, 8), 16) * 0.01).toFixed(2) + "V";
        str = str.substring(8);
        break;
      case 0x0586:// gyroscope
        var gyroscope = {};
        gyroscope.x = (parseShort(str.substring(4, 8), 16) * 0.01).toFixed(2) + "°/s";
        gyroscope.y = (parseShort(str.substring(8, 12), 16) * 0.01).toFixed(2) + "°/s";
        gyroscope.z = (parseShort(str.substring(12, 16), 16) * 0.01).toFixed(2) + "°/s";
        myObj.gyroscope = gyroscope;
        str = str.substring(16);
        break;
      case 0x0902:// magnetometer x
        magnetometer.x = (parseShort(str.substring(4, 8), 16) * 0.01).toFixed(2) + "μT";
        str = str.substring(8);
        break;
      case 0x0a02:// magnetometer y
        magnetometer.y = (parseShort(str.substring(4, 8), 16) * 0.01).toFixed(2) + "μT";
        str = str.substring(8);
        break;
      case 0x0b02:// magnetometer z
        magnetometer.z = (parseShort(str.substring(4, 8), 16) * 0.01).toFixed(2) + "μT";
        str = str.substring(8);
        break;
      default:
        str = str.substring(7);
        break;
    }
  }
  if(Object.getOwnPropertyNames(environment).length > 0) {
    myObj.environment = environment;
  }
  if(Object.getOwnPropertyNames(magnetometer).length > 0) {
    myObj.magnetometer = magnetometer;
  }
  return myObj;
}

i would like to add that i integrate a postgresql DB and no problem…
an futhermore i have done some query in this DB with grafana without troubles too…

someone could help me?

Resolved!

The backslash provide from “escaping” …
The solution is to unescape the message .

The Json.parse() function do that but in my case i have a Json object into a Json object.

“ojectJSON” from de payload codec is integrate into the “Json object” from the Application server.
So double level of integration => necessary to double the decoder.

Now i’m asking if its necessary that the payload codec encode a json object as its done by the application server?

In fact its not resolved !!
And i understand better the problem:
the problem come from the type “string” into the “objectJSON” key into the mqtt message?
look at this:

"objectJSON":"
			{
			\"DecodeDataHex\":\"0802016e07685d06732577026700cd040202e2\",
			\"DecodeDataObj\":
				{
				\"battery\":\"3.66V\",
				\"environment\":
						{
						\"barometer\":\"959.10hPa\",
						\"gasResistance\":\"7.38KΩ\",
						\"humidity\":\"46.5% RH\",
						\"temperature\":\"20.50°C\"
						}
				}
			}",		

Its a “string” .
Im waiting for json “key” and “field”.
This not permit me to use JSON.parse() on my message. SyntaxError: unexpected token at…

So could i use a other function to parse this message into a object?
Sorry but i’m not a pro in JavaScript so i cant decompose this message until get what i want.
Futhermore i need to integrate this message into a application where i haven’t the hand on all the feature.
and this application decode only json object .

The ObjectJson is not into Json format.
Just to add that remove double quote a the start and end of the field, resolve the problem…

sure, because you add the special chars KΩ,°C etc. in the your decoder.
if you want to operate with values, remove them while you forming object

1 Like

Thanks to read, and help, but no its doesnt concern “units”.
here some example with and witout units:

{ 
"battery":"3.66",
"environment": 	{
	"barometer":"959.10dqzdqddzq",
	"gasResistance":"7.38dqzdqzzd",
	"humidity":"46.5",
	"temperature":"20.50xxxxxxx"
		},

    "OtherJsonObject": {
        "someKey": "somefield",
        "someOthergroup": {
            "AnArray": [
                { "temp1WithUnit": "2.56°C", "humidity1WithUnit": "56.3%rh" },
                { "temp2WithoutUnit": "2.56", "humidity2WithoutUnit": "56.3" },
                { "temp3WithAny": "2.56dqjndq", "humidity3WithAny": "56qjdksqd" }
            		]
       			 }
    			}
	
}

No the problem came from the manner as the Object is stringify()
I have asked at RAK to modify their codec to obtain a simplify object . They dit it!
now the object have less hierarchy and i’m waiting to get somethings like this:

{
"Jsonobject": 	{
	"barometer":"959.10",
	"battery":"3.66",
	"gasResistance":"1.68",
	"humidity":"46.5",
	"temperature":"20.50"
		}
}

but i get this:

{
"Jsonobject": "	{
	\"barometer\":\"959.10\",
	\"gasResistance\":\"7.38\",
	\"humidity\":\"46.5\",
	\"temperature\":\"20.50\"
	}"
}

so some: double quote and backslash too much …

I suppose you getting this after inserting and selecting json from postgresql? or what? please be more informative.
anyway.
did you read this ? PostgreSQL: Documentation: 16: 9.16. JSON Functions and Operators
also try to use chirpstack + influxdb + grafana this works fine for me.

Blockquote
you getting this after inserting and selecting json from postgresql?

No i getting it from MQTT subscription.

i already use chirpstack + postgres + grafana and it works fine!
(just a little problem with the float number and the tofixed() function I dont know why but the decimal number is not fixe and it’s auto reducing into postgres, its an other topics that i would like to talk later)

But here its with MQTT.
and what i’m seeing is:
i will try to resume how i imagine the transfert :

Loranode give :
base64 “data” => payloadCodec => MyObjectData => Chirpstack_NS => stringify(MyobjecData) => MQTTbroker => message = “SomeGatewayDataInstring+MyobjectDataInString” ==> Mqtt client => JSON.pars(message) => result : SomeGatewayData Ok ; Myobjetdata = Corrupt.

"error = unexpected token at position … " because i get a double quote and backslash just behind the “key” "objectJSON"
As if the object was parse (stringify) twice…

I hope to be clearer.

that’s why you got a backslashed double quotes inside msg. because it is string type after your manipulations instead json object.

ok cool it permit me to understand better…
now…

Blockquote because it is string type after your manipulations instead json object.

no no , i manipulate nothing… the message that i publish is the result of :
mosquitto_sub -t -v /#
In the shell of the applicationserver’s host

There are no parse anywhere…