Getting issue while decoding chirpstack stream frame and event when taken directly from redis database

@brocaar : Can you please help in this case for decoding the live stream frame and events from the redis database. Its critical requirement for us to decode the frames and events and display. Can you please help in this urgent case.

This is the live event data format: ```
{
“1716454763380-0”: {
“up”: “\n\u0018@1\u0001\n\u0001\u0002\b\u0002p\u0001\b$\u000f\u0012\u0012\b\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a|\n\u00100080000000021494\u0010\u00030\u0001=\u0000\u0000\u0000A@\u0002b\u0000j\u0004v\u0005\b\u0013z\u001b\n\u0010region_config_id\u0012\u0007us915_1z\u001b\n\u0012region_common_name\u0012\u0005US915\u0001\u0002\u0001\f\bꊼ\u0006\u0010\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\b늼\u0006\u0010뒵\u0001@\u0001H\u0001”
}
}

Following this link: Frame logging - ChirpStack open-source LoRaWAN® Network Server documentation

What problem are you having?

It looks like the doc page has the wrong URL, as those protos were moved into the stream/ directory in a recent release:

Otherwise, you’ll need to be more specific on the problem you’re experiencing.

This is my code to read stream frame/event from redis database:

import redis
import json

Connect to Redis

redis_client = redis.StrictRedis(host=‘localhost’, port=6379, db=0)

frame_or_event = ‘event’
device_pattern = f"device:{{*}}:stream:{frame_or_event}"
keys = redis_client.keys(device_pattern)

final_dict = {}
device_data = {}

def parse_data(stream_data,device_eui):
parsed_data =
for device, events in stream_data:
device_str = device.decode()
device_data = {f’{device_eui} - events’: }

    for event_id, event_dict in events:
        event_id_str = event_id.decode()
        event_data = {}
        
        for key, value in event_dict.items():
            key_str = key.decode()
            value_decoded = value.decode('utf-8', errors='ignore') # Decode and ignore errors
            
            event_data[key_str] = value_decoded
        
        device_data[f'{device_eui} - events'].append({event_id_str: event_data})
    
    parsed_data.append(device_data)
    final_dict.update(device_data)
return parsed_data,final_dict

for key in keys:
try:
if isinstance(key, bytes):
key_str = key.decode()
else:
key_str = key
device_eui = key_str.split(‘:’)[1].strip(‘{}’)
key_type = redis_client.type(key)
if key_type == b’stream’:
streams = redis_client.xread({key: ‘0’}, count=10)
print(streams)
parsed_stream_data,final_dict = parse_data(streams,device_eui)
except Exception as e:
print(f"Error processing key {key}: {e}")

This is the response I am getting while reading the response:

{
“data”: {
“request_id”: “admin-1716460477.5629792”,
“response”: {
“a840415eb18492f2 - frame”: [
{
“1716454763380-0”: {
“up”: “\n\u0018@1\u0001\n\u0001\u0002\b\u0002p\u0001\b$\u000f\u0012\u0012\b\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a|\n\u00100080000000021494\u0010\u00030\u0001=\u0000\u0000\u0000A@\u0002b\u0000j\u0004v\u0005\b\u0013z\u001b\n\u0010region_config_id\u0012\u0007us915_1z\u001b\n\u0012region_common_name\u0012\u0005US915\u0001\u0002\u0001\f\bꊼ\u0006\u0010\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\b늼\u0006\u0010뒵\u0001@\u0001H\u0001”
}
},
{
“1716455363362-0”: {
“up”: “\n\u0018@1\u0001\u000b\u0001\u0002\b\u0002l\u0001\b5\u0004!\u0012\u0012\b۾\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a~\n\u00100080000000021494\u0010\u00030\u0001=\u0000\u0000\u0010A@\u0005H\u0001b\u0000j\u0004\u001ftz\u001b\n\u0010region_config_id\u0012\u0007us915_1z\u001b\n\u0012region_common_name\u0012\u0005US915\u0001\u0002\u0001\f\b\u0006\u0010\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\bÏ\u0006\u0010Ĭ\u0001@\u0001H\u0001”
}
},
{
“1716455963354-0”: {
“up”: “\n\u0018@1\u0001\f\u0001\u0002\b\u0002l\u0001\bɃT\u0012\u0012\bց\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001az\n\u00100080000000021494\u0010\u00020\u0001=\u0000\u0000\u0010Ab\u0000j\u0004BSz\u001b\n\u0010region_config_id\u0012\u0007us915_1z\u001b\n\u0012region_common_name\u0012\u0005US915\u0001\u0002\u0001\f\b\u0006\u0010\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\b\u0006\u0010\u0001@\u0001H\u0001”
}
},
{
“1716456563333-0”: {
“up”: “\n\u0018@1\u0001\r\u0001\u0002\b\u0002k\u0001\b)x\u0012\u0012\b\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a{\n\u00100080000000021494\u0010\t0\u0001=33#A@\u0003b\u0000j\u0004Ne3z\u001b\n\u0010region_config_id\u0012\u0007us915_1z\u001b\n\u0012region_common_name\u0012\u0005US915\u0001\u0002\u0001\f\b򘼲\u0006\u0010\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\b󘼲\u0006\u0010ྞ\u0001@\u0001H\u0001”
}
},
{
“1716457163352-0”: {
“up”: “\n\u0018@1\u0001\u000e\u0001\u0002\b\u0002l\u0001\b_\t}\u0012\u0012\bʯ\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a~\n\u00100080000000021494\u0010\u00010\u0001=\u0000\u0000@@\u0006H\u0001b\u0000j\u0004\u0005\u0011\u001cz\u001b\n\u0010region_config_id\u0012\u0007us915_1z\u001b\n\u0012region_common_name\u0012\u0005US915\u0001\u0002\u0001\f\bʝ\u0006\u0010Ƽ\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\b˝\u0006\u0010˧\u0001@\u0001H\u0001”
}
},
{
“1716457763172-0”: {
“up”: “\n\u0018@1\u0001\u000f\u0001\u0002\b\u0002k\u0001\b4,#\u0012\u0012\b\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a|\n\u00100080000000021494\u0010\u00030\u0001=\u0000\u0000\u0018A@\u0001b\u0000j\u0004(Ԫz\u001b\n\u0012region_common_name\u0012\u0005US915z\u001b\n\u0010region_config_id\u0012\u0007us915_1\u0001\u0002\u0001\f\b\u0006\u0010Ĩ\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\u000b\b\u0006\u0010Q@\u0001H\u0001”
}
},
{
“1716458363349-0”: {
“up”: “\n\u0018@1\u0001\u0010\u0001\u0002\b\u0002m\u0001\b%E,\u0012\u0012\b۾\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a~\n\u00100080000000021494\u0010\u00030\u0001=\u0000\u0000(A@\u0005H\u0001b\u0000j\u0004Lz\u001b\n\u0012region_common_name\u0012\u0005US915z\u001b\n\u0010region_config_id\u0012\u0007us915_1\u0001\u0002\u0001\f\b\u0006\u0010\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\b\u0006\u0010\u0001@\u0001H\u0001”
}
},
{
“1716458963310-0”: {
“up”: “\n\u0018@1\u0001\u0011\u0001\u0002\b\u0002n\u0001\b3\u0012\f\u0012\u0012\b\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a|\n\u00100080000000021494\u0010\u00020\u0001=33\u0013A@\u0003b\u0000j\u0004pZz\u001b\n\u0012region_common_name\u0012\u0005US915z\u001b\n\u0010region_config_id\u0012\u0007us915_1\u0001\u0002\u0001\f\bҫ\u0006\u0010\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\bӫ\u0006\u0010\u0001@\u0001H\u0001”
}
},
{
“1716459563296-0”: {
“up”: “\n\u0018@1\u0001\u0012\u0001\u0002\b\u0002m\u0001\b֎A\u0012\u0012\b\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a~\n\u00100080000000021494\u0010μ\u00010\u0001=33\u0013A@\u0004H\u0001b\u0000j\u0004\u001e\u001f+z\u001b\n\u0012region_common_name\u0012\u0005US915z\u001b\n\u0010region_config_id\u0012\u0007us915_1\u0001\u0002\u0001\f\b\u0006\u0010Ɂԯ\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\b\u0006\u0010\u0001@\u0001H\u0001”
}
},
{
“1716460163283-0”: {
“up”: “\n\u0018@1\u0001\u0013\u0001\u0002\b\u0002k\u0001\b=fG\u0012\u0012\b\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a|\n\u00100080000000021494\u0010\u00030\u0001=\u0000\u0000@@\u0002b\u0000j\u00046z\u001b\n\u0012region_common_name\u0012\u0005US915z\u001b\n\u0010region_config_id\u0012\u0007us915_1\u0001\u0002\u0001\f\b\u0006\u0010␫\u0003 \u0002*\b01b686312\u0010a840415eb18492f2:\f\b\u0006\u0010̆\u0001@\u0001H\u0001”
}
}
]
},
“status_code”: 200
},
“error”: null
}

This is the code I am using to decode the response

import frames_pb2
import protobuf

Sample data

data = {
“1716348523332-0”: { “up”: “\n\u0018@{S\u0001-\u0000\u0002\t\u0002\u0001\th\u000f7\u0012\u0012\b\u0003\u0012\n\u001a\b\b\u0007\u0010\u0007(\u0001\u001a~\n\u00100080000000021494\u0010\u00010\u0001=\u0000\u00000A@\u0004H\u0001b\u0000j\u0004{z\u001b\n\u0010region_config_id\u0012\u0007us915_1z\u001b\n\u0012region_common_name\u0012\u0005US915\u0001\u0002\u0001\f\b̵\u0006\u0010\u0003 \u0002*\b0153e47b2\u0010a84041baf184b029:\f\b̵\u0006\u0010\u0001@\u0001H\u0001” }
}

Extract the encoded string

encoded_str = data[“1716348523332-0”][“up”]

decoded_payload = encoded_str.encode().decode(‘unicode_escape’).encode(‘latin1’)

print(f"decoded_payload:{decoded_payload}")

uplink_event = frames_pb2.UplinkFrameLog()
uplink_event.ParseFromString(decoded_payload)
print(uplink_event)

Getting this error:
Traceback (most recent call last):
File “event_decoder.py”, line 16, in
uplink_event.ParseFromString(payload)
google.protobuf.message.DecodeError: Error parsing message with type ‘streams.UplinkFrameLog’

@brocaar: Any help on this

Please see the code examples here: chirpstack/examples at master · chirpstack/chirpstack · GitHub