Downlink fcount equals 0 issue

Hi,

We are facing a strange issue with downlink message :

  • We create using GRPC interface a new OTAA device
  • The device joins and sends a confirmed uplink message
  • Ack is received on device
  • We create using GRPC a enqueued downlink message
  • The device send a new confirmed uplink message
  • The downlink ack is sent with enqueued downlinik payload

This behaviour works from a long time on our test chirpstack installation.
We upgraded to the last version from today and we are facing a strange issue with the second downlink message (with payload) having a fcount equals 0. So our device discard the message.

We didn’t see this issue before with previous versions and it worked very well.
Here is some frame logs from web interface with a correct case and a bad one.

[
    {
        "type": "txack",
        "payload": {
            "applicationID": "1",
            "applicationName": "RAK2245",
            "deviceName": "DUT-2C522B3424A397D0",
            "devEUI": "LFIrNCSjl9A=",
            "fCnt": 0,
            "tags": {},
            "gatewayID": "q83v//6rze8=",
            "txInfo": {
                "gatewayID": null,
                "frequency": 868100000,
                "power": 14,
                "modulation": "LORA",
                "loRaModulationInfo": {
                    "bandwidth": 125,
                    "spreadingFactor": 7,
                    "codeRate": "4/5",
                    "polarizationInversion": true
                },
                "board": 0,
                "antenna": 0,
                "timing": "DELAY",
                "delayTimingInfo": {
                    "delay": "1s"
                },
                "context": "BMXmWw=="
            },
            "publishedAt": "2021-09-01T11:45:42.426440922Z"
        }
    },
    {
        "type": "up",
        "payload": {
            "applicationID": "1",
            "applicationName": "RAK2245",
            "deviceName": "DUT-2C522B3424A397D0",
            "devEUI": "LFIrNCSjl9A=",
            "rxInfo": [
                {
                    "gatewayID": "q83v//6rze8=",
                    "time": null,
                    "timeSinceGPSEpoch": null,
                    "rssi": -11,
                    "loRaSNR": 10.8,
                    "channel": 0,
                    "rfChain": 1,
                    "board": 0,
                    "antenna": 0,
                    "location": {
                        "latitude": 0,
                        "longitude": 0,
                        "altitude": 0,
                        "source": "UNKNOWN",
                        "accuracy": 0
                    },
                    "fineTimestampType": "NONE",
                    "context": "BMXmWw==",
                    "uplinkID": "3V4sVDTtQUKQS4Dhs3f/Dg==",
                    "crcStatus": "CRC_OK"
                }
            ],
            "txInfo": {
                "frequency": 868100000,
                "modulation": "LORA",
                "loRaModulationInfo": {
                    "bandwidth": 125,
                    "spreadingFactor": 7,
                    "codeRate": "4/5",
                    "polarizationInversion": false
                }
            },
            "adr": false,
            "dr": 5,
            "fCnt": 2,
            "fPort": 223,
            "data": "+////+wA/P///+0A/P///+0A",
            "objectJSON": "",
            "tags": {},
            "confirmedUplink": true,
            "devAddr": "B41yaQ==",
            "publishedAt": "2021-09-01T11:45:42.312992136Z"
        }
    },
    {
        "type": "up",
        "payload": {
            "applicationID": "1",
            "applicationName": "RAK2245",
            "deviceName": "DUT-2C522B3424A397D0",
            "devEUI": "LFIrNCSjl9A=",
            "rxInfo": [
                {
                    "gatewayID": "q83v//6rze8=",
                    "time": null,
                    "timeSinceGPSEpoch": null,
                    "rssi": -11,
                    "loRaSNR": 10,
                    "channel": 0,
                    "rfChain": 1,
                    "board": 0,
                    "antenna": 0,
                    "location": {
                        "latitude": 0,
                        "longitude": 0,
                        "altitude": 0,
                        "source": "UNKNOWN",
                        "accuracy": 0
                    },
                    "fineTimestampType": "NONE",
                    "context": "BJHxAw==",
                    "uplinkID": "Iw81epXpSdy0rFjl4oL3NA==",
                    "crcStatus": "CRC_OK"
                }
            ],
            "txInfo": {
                "frequency": 868100000,
                "modulation": "LORA",
                "loRaModulationInfo": {
                    "bandwidth": 125,
                    "spreadingFactor": 7,
                    "codeRate": "4/5",
                    "polarizationInversion": false
                }
            },
            "adr": false,
            "dr": 5,
            "fCnt": 1,
            "fPort": 223,
            "data": "/f/+/+0A/P///+wA/P///+wA",
            "objectJSON": "",
            "tags": {},
            "confirmedUplink": true,
            "devAddr": "B41yaQ==",
            "publishedAt": "2021-09-01T11:45:38.925019720Z"
        }
    },
    {
        "type": "join",
        "payload": {
            "applicationID": "1",
            "applicationName": "RAK2245",
            "deviceName": "DUT-2C522B3424A397D0",
            "devEUI": "LFIrNCSjl9A=",
            "devAddr": "B41yaQ==",
            "rxInfo": [
                {
                    "gatewayID": "q83v//6rze8=",
                    "time": null,
                    "timeSinceGPSEpoch": null,
                    "rssi": -11,
                    "loRaSNR": 10,
                    "channel": 0,
                    "rfChain": 1,
                    "board": 0,
                    "antenna": 0,
                    "location": {
                        "latitude": 0,
                        "longitude": 0,
                        "altitude": 0,
                        "source": "UNKNOWN",
                        "accuracy": 0
                    },
                    "fineTimestampType": "NONE",
                    "context": "BJHxAw==",
                    "uplinkID": "Iw81epXpSdy0rFjl4oL3NA==",
                    "crcStatus": "CRC_OK"
                }
            ],
            "txInfo": {
                "frequency": 868100000,
                "modulation": "LORA",
                "loRaModulationInfo": {
                    "bandwidth": 125,
                    "spreadingFactor": 7,
                    "codeRate": "4/5",
                    "polarizationInversion": false
                }
            },
            "dr": 5,
            "tags": {},
            "publishedAt": "2021-09-01T11:45:38.919582439Z"
        }
    }
]

Some lorawan KO frames

[
    {
        "txInfo": {
            "frequency": 868100000,
            "power": 14,
            "modulation": "LORA",
            "loRaModulationInfo": {
                "bandwidth": 125,
                "spreadingFactor": 7,
                "codeRate": "4/5",
                "polarizationInversion": true
            },
            "board": 0,
            "antenna": 0,
            "timing": "DELAY",
            "delayTimingInfo": {
                "delay": "1s"
            },
            "context": "FACziw=="
        },
        "phyPayload": {
            "mhdr": {
                "mType": "UnconfirmedDataDown",
                "major": "LoRaWANR1"
            },
            "macPayload": {
                "fhdr": {
                    "devAddr": "06aab0db",
                    "fCtrl": {
                        "adr": true,
                        "adrAckReq": false,
                        "ack": true,
                        "fPending": false,
                        "classB": false
                    },
                    "fCnt": 0,
                    "fOpts": null
                },
                "fPort": 224,
                "frmPayload": [
                    {
                        "bytes": "yg70jBSVIZl9Y0M3Z/Uu4gYxQ1o="
                    }
                ]
            },
            "mic": "28292ef7"
        }
    },
    {
        "rxInfo": [
            {
                "gatewayID": "q83v//6rze8=",
                "time": null,
                "timeSinceGPSEpoch": null,
                "rssi": -10,
                "loRaSNR": 11.5,
                "channel": 0,
                "rfChain": 1,
                "board": 0,
                "antenna": 0,
                "location": {
                    "latitude": 0,
                    "longitude": 0,
                    "altitude": 0,
                    "source": "UNKNOWN",
                    "accuracy": 0
                },
                "fineTimestampType": "NONE",
                "context": "FACziw==",
                "uplinkID": "jJMU/MFEQgGE6Ra+h96qew==",
                "crcStatus": "CRC_OK"
            }
        ],
        "txInfo": {
            "frequency": 868100000,
            "modulation": "LORA",
            "loRaModulationInfo": {
                "bandwidth": 125,
                "spreadingFactor": 7,
                "codeRate": "4/5",
                "polarizationInversion": false
            }
        },
        "phyPayload": {
            "mhdr": {
                "mType": "ConfirmedDataUp",
                "major": "LoRaWANR1"
            },
            "macPayload": {
                "fhdr": {
                    "devAddr": "06aab0db",
                    "fCtrl": {
                        "adr": false,
                        "adrAckReq": false,
                        "ack": false,
                        "fPending": false,
                        "classB": false
                    },
                    "fCnt": 2,
                    "fOpts": null
                },
                "fPort": 223,
                "frmPayload": [
                    {
                        "bytes": "m163aVUfqZA8VELpKO3ajV7y"
                    }
                ]
            },
            "mic": "a4585c61"
        }
    },
    {
        "rxInfo": [
            {
                "gatewayID": "q83v//6rze8=",
                "time": null,
                "timeSinceGPSEpoch": null,
                "rssi": -10,
                "loRaSNR": 10,
                "channel": 0,
                "rfChain": 1,
                "board": 0,
                "antenna": 0,
                "location": {
                    "latitude": 0,
                    "longitude": 0,
                    "altitude": 0,
                    "source": "UNKNOWN",
                    "accuracy": 0
                },
                "fineTimestampType": "NONE",
                "context": "E8y02w==",
                "uplinkID": "uYVzVKNWQVGBkf1T1oIf1g==",
                "crcStatus": "CRC_OK"
            }
        ],
        "txInfo": {
            "frequency": 868100000,
            "modulation": "LORA",
            "loRaModulationInfo": {
                "bandwidth": 125,
                "spreadingFactor": 7,
                "codeRate": "4/5",
                "polarizationInversion": false
            }
        },
        "phyPayload": {
            "mhdr": {
                "mType": "ConfirmedDataUp",
                "major": "LoRaWANR1"
            },
            "macPayload": {
                "fhdr": {
                    "devAddr": "06aab0db",
                    "fCtrl": {
                        "adr": false,
                        "adrAckReq": false,
                        "ack": false,
                        "fPending": false,
                        "classB": false
                    },
                    "fCnt": 1,
                    "fOpts": null
                },
                "fPort": 223,
                "frmPayload": [
                    {
                        "bytes": "ZJLM82oMDU9L1ZHZhbuOuhY8"
                    }
                ]
            },
            "mic": "91fe8bc6"
        }
    },
    {
        "txInfo": {
            "frequency": 868100000,
            "power": 14,
            "modulation": "LORA",
            "loRaModulationInfo": {
                "bandwidth": 125,
                "spreadingFactor": 7,
                "codeRate": "4/5",
                "polarizationInversion": true
            },
            "board": 0,
            "antenna": 0,
            "timing": "DELAY",
            "delayTimingInfo": {
                "delay": "5s"
            },
            "context": "E2w7sw=="
        },
        "phyPayload": {
            "mhdr": {
                "mType": "JoinAccept",
                "major": "LoRaWANR1"
            },
            "macPayload": {
                "bytes": "ANsi6Bfr6qvcW3JpKLZ/4+cg9/BsZWgmKGBqvQ=="
            },
            "mic": "95da6c8f"
        }
    },
    {
        "rxInfo": [
            {
                "gatewayID": "q83v//6rze8=",
                "time": null,
                "timeSinceGPSEpoch": null,
                "rssi": -9,
                "loRaSNR": 9,
                "channel": 0,
                "rfChain": 1,
                "board": 0,
                "antenna": 0,
                "location": {
                    "latitude": 0,
                    "longitude": 0,
                    "altitude": 0,
                    "source": "UNKNOWN",
                    "accuracy": 0
                },
                "fineTimestampType": "NONE",
                "context": "E2w7sw==",
                "uplinkID": "LI/Jvv/aR5yGE4zKVLMkEA==",
                "crcStatus": "CRC_OK"
            }
        ],
        "txInfo": {
            "frequency": 868100000,
            "modulation": "LORA",
            "loRaModulationInfo": {
                "bandwidth": 125,
                "spreadingFactor": 7,
                "codeRate": "4/5",
                "polarizationInversion": false
            }
        },
        "phyPayload": {
            "mhdr": {
                "mType": "JoinRequest",
                "major": "LoRaWANR1"
            },
            "macPayload": {
                "joinEUI": "389fffc6ad6f92f5",
                "devEUI": "389fffc6ad6f92f5",
                "devNonce": 28232
            },
            "mic": "d966e240"
        }
    }
]

Some device OK frames

[
    {
        "type": "txack",
        "payload": {
            "applicationID": "1",
            "applicationName": "RAK2245",
            "deviceName": "DUT-9429772D9A24BF4F",
            "devEUI": "lCl3LZokv08=",
            "fCnt": 1,
            "tags": {},
            "gatewayID": "q83v//6rze8=",
            "txInfo": {
                "gatewayID": null,
                "frequency": 868100000,
                "power": 14,
                "modulation": "LORA",
                "loRaModulationInfo": {
                    "bandwidth": 125,
                    "spreadingFactor": 7,
                    "codeRate": "4/5",
                    "polarizationInversion": true
                },
                "board": 0,
                "antenna": 0,
                "timing": "DELAY",
                "delayTimingInfo": {
                    "delay": "1s"
                },
                "context": "C9sx6w=="
            },
            "publishedAt": "2021-09-01T11:47:41.263349519Z"
        }
    },
    {
        "type": "up",
        "payload": {
            "applicationID": "1",
            "applicationName": "RAK2245",
            "deviceName": "DUT-9429772D9A24BF4F",
            "devEUI": "lCl3LZokv08=",
            "rxInfo": [
                {
                    "gatewayID": "q83v//6rze8=",
                    "time": null,
                    "timeSinceGPSEpoch": null,
                    "rssi": -10,
                    "loRaSNR": 9.2,
                    "channel": 0,
                    "rfChain": 1,
                    "board": 0,
                    "antenna": 0,
                    "location": {
                        "latitude": 0,
                        "longitude": 0,
                        "altitude": 0,
                        "source": "UNKNOWN",
                        "accuracy": 0
                    },
                    "fineTimestampType": "NONE",
                    "context": "C9sx6w==",
                    "uplinkID": "CGukCPgCS0qYLqsVBcz3fw==",
                    "crcStatus": "CRC_OK"
                }
            ],
            "txInfo": {
                "frequency": 868100000,
                "modulation": "LORA",
                "loRaModulationInfo": {
                    "bandwidth": 125,
                    "spreadingFactor": 7,
                    "codeRate": "4/5",
                    "polarizationInversion": false
                }
            },
            "adr": false,
            "dr": 5,
            "fCnt": 2,
            "fPort": 223,
            "data": "/P/8/+0A/P/+/+4A/P/+/+4A",
            "objectJSON": "",
            "tags": {},
            "confirmedUplink": true,
            "devAddr": "B70+Eg==",
            "publishedAt": "2021-09-01T11:47:41.146857410Z"
        }
    },
    {
        "type": "up",
        "payload": {
            "applicationID": "1",
            "applicationName": "RAK2245",
            "deviceName": "DUT-9429772D9A24BF4F",
            "devEUI": "lCl3LZokv08=",
            "rxInfo": [
                {
                    "gatewayID": "q83v//6rze8=",
                    "time": null,
                    "timeSinceGPSEpoch": null,
                    "rssi": -9,
                    "loRaSNR": 9.2,
                    "channel": 0,
                    "rfChain": 1,
                    "board": 0,
                    "antenna": 0,
                    "location": {
                        "latitude": 0,
                        "longitude": 0,
                        "altitude": 0,
                        "source": "UNKNOWN",
                        "accuracy": 0
                    },
                    "fineTimestampType": "NONE",
                    "context": "C6djyw==",
                    "uplinkID": "3SIyngkOR/6fGh0cSN4wNw==",
                    "crcStatus": "CRC_OK"
                }
            ],
            "txInfo": {
                "frequency": 868100000,
                "modulation": "LORA",
                "loRaModulationInfo": {
                    "bandwidth": 125,
                    "spreadingFactor": 7,
                    "codeRate": "4/5",
                    "polarizationInversion": false
                }
            },
            "adr": false,
            "dr": 5,
            "fCnt": 1,
            "fPort": 223,
            "data": "/f/+/+0A/f/+/+0A/f/+/+0A",
            "objectJSON": "",
            "tags": {},
            "confirmedUplink": true,
            "devAddr": "B70+Eg==",
            "publishedAt": "2021-09-01T11:47:37.768550564Z"
        }
    },
    {
        "type": "join",
        "payload": {
            "applicationID": "1",
            "applicationName": "RAK2245",
            "deviceName": "DUT-9429772D9A24BF4F",
            "devEUI": "lCl3LZokv08=",
            "devAddr": "B70+Eg==",
            "rxInfo": [
                {
                    "gatewayID": "q83v//6rze8=",
                    "time": null,
                    "timeSinceGPSEpoch": null,
                    "rssi": -9,
                    "loRaSNR": 9.2,
                    "channel": 0,
                    "rfChain": 1,
                    "board": 0,
                    "antenna": 0,
                    "location": {
                        "latitude": 0,
                        "longitude": 0,
                        "altitude": 0,
                        "source": "UNKNOWN",
                        "accuracy": 0
                    },
                    "fineTimestampType": "NONE",
                    "context": "C6djyw==",
                    "uplinkID": "3SIyngkOR/6fGh0cSN4wNw==",
                    "crcStatus": "CRC_OK"
                }
            ],
            "txInfo": {
                "frequency": 868100000,
                "modulation": "LORA",
                "loRaModulationInfo": {
                    "bandwidth": 125,
                    "spreadingFactor": 7,
                    "codeRate": "4/5",
                    "polarizationInversion": false
                }
            },
            "dr": 5,
            "tags": {},
            "publishedAt": "2021-09-01T11:47:37.763945553Z"
        }
    }
]

Also chirpstack corresponding logs for gateway bridge, network server and application server with colors from docker compose output https://we.tl/t-NWy39WZUKQ
(can’t attach it directly to the board)

We are open to try and debug stuff.

Many thanks for any help.