feat(gateway): implement BLE-to-MQTT gateway
Implement Gateway class that discovers Nordic Thingy:52 nodes via BLE
and publishes sensor data to MQTT broker on each notification received.
- Automatic node discovery via BLE service UUID (ef680100)
- GATT notifications for temperature, humidity and CO2
- Publish immediately on reception to {gateway_id}/{mac}/update
- Connection timeout to avoid blocking on unreachable nodes
- Disconnection detection and automatic reintegration into scan
- Logging with DEBUG/INFO/WARNING/ERROR levels
Assisted-by: Claude:claude-sonnet-4-6 — debugging BLE parallel connections (BlueZ InProgress error), GATT UUID discovery (ef680100 vs ef680200), byte decoding for temperature/humidity/CO2, async connection timeout implementation
This commit is contained in:
DjeAvd
@@ -1,242 +1,160 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import csv
|
import logging
|
||||||
import os
|
import os
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from bleak import BleakClient, BleakScanner
|
from bleak import BleakClient, BleakScanner
|
||||||
import paho.mqtt.client as mqtt
|
import paho.mqtt.client as mqtt
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# Logging — use DEBUG for development, INFO for normal operation
|
||||||
# GATT service and characteristic UUIDs for the Nordic Thingy:52
|
logging.basicConfig(
|
||||||
# These UUIDs are proprietary to Nordic Semiconductor and are used to
|
level=logging.INFO,
|
||||||
# identify the environmental sensor service and its characteristics over BLE.
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||||
#
|
datefmt="%Y-%m-%dT%H:%M:%S"
|
||||||
# THINGY_SERVICE_UUID corresponds to the Configuration service (ef680100),
|
)
|
||||||
# which is the UUID advertised by the Thingy:52 in its BLE advertising packets.
|
log = logging.getLogger(__name__)
|
||||||
# Note: the Environment service (ef680200) is available after connection
|
|
||||||
# but is not advertised, so it cannot be used for device discovery.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
THINGY_SERVICE_UUID = "ef680100-9b35-4933-9b10-52ffa9740042"
|
|
||||||
UUID_TEMP = "ef680201-9b35-4933-9b10-52ffa9740042"
|
|
||||||
UUID_CO2 = "ef680204-9b35-4933-9b10-52ffa9740042"
|
|
||||||
UUID_HUMIDITY = "ef680203-9b35-4933-9b10-52ffa9740042"
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# MQTT broker configuration
|
|
||||||
# The broker runs locally on the Raspberry Pi (Mosquitto).
|
|
||||||
# Other teams subscribe to the topics published here.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
MQTT_BROKER = "localhost"
|
|
||||||
MQTT_PORT = 1883
|
|
||||||
|
|
||||||
# Publishing interval in seconds (300 = every 5 minutes in production)
|
class Gateway:
|
||||||
INTERVAL = 300
|
"""BLE to MQTT gateway for Nordic Thingy:52 sensor nodes."""
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
def __init__(self, config: dict):
|
||||||
# Room identification
|
self.gateway_id = config["gateway_id"]
|
||||||
# The operator enters the room ID at startup. This value is used to name
|
self.mqtt_broker = config["mqtt"]["broker"]
|
||||||
# the output files and structure the MQTT topics accordingly.
|
self.mqtt_port = config["mqtt"]["port"]
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
print("=== Gateway IoT - HES-SO ===")
|
# GATT UUIDs loaded from config
|
||||||
ROOM_ID = input("Which room are you in? (e.g. C1, A2, B5) : ").strip().upper()
|
# ef680100 is advertised in BLE packets — used for discovery
|
||||||
print(f"Room configured : {ROOM_ID}\n")
|
# environment service (ef680200) is only accessible after connection
|
||||||
|
self.service_uuid = config["ble"]["service_uuid"]
|
||||||
|
self.uuid_temp = config["ble"]["characteristics"]["temperature"]
|
||||||
|
self.uuid_co2 = config["ble"]["characteristics"]["co2"]
|
||||||
|
self.uuid_humidity= config["ble"]["characteristics"]["humidity"]
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# Runtime state
|
||||||
# Output file paths
|
# latest : last known sensor values per MAC address
|
||||||
# One CSV and one JSON file are created per room, named after the room ID.
|
# connecting: MACs currently being connected (avoids duplicate attempts)
|
||||||
# The CSV is intended for local analysis (e.g. Excel).
|
self.latest = {}
|
||||||
# The JSON follows the format expected by the database team.
|
self.connecting = set()
|
||||||
# ---------------------------------------------------------------------------
|
self.scanner = None
|
||||||
BASE_PATH = "/home/pi/gateway"
|
|
||||||
CSV_FILE = f"{BASE_PATH}/data_{ROOM_ID}.csv"
|
|
||||||
JSON_FILE = f"{BASE_PATH}/data_{ROOM_ID}.json"
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# MQTT client setup
|
||||||
# Runtime state
|
self.mqttc = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2)
|
||||||
# discovered : maps each device MAC address to its assigned node ID
|
self.mqttc.connect(self.mqtt_broker, self.mqtt_port)
|
||||||
# latest : stores the most recent sensor values for each device
|
self.mqttc.loop_start()
|
||||||
# connecting : tracks MAC addresses currently being connected to
|
|
||||||
# to prevent duplicate connection attempts during BLE scanning
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
discovered = {}
|
|
||||||
latest = {}
|
|
||||||
connecting = set()
|
|
||||||
thingy_counter = [0]
|
|
||||||
scanner = None
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
log.info(f"Gateway ID : {self.gateway_id}")
|
||||||
# MQTT client setup
|
log.info(f"MQTT broker: {self.mqtt_broker}:{self.mqtt_port}")
|
||||||
# The client connects to the local Mosquitto broker and starts its
|
log.info("MQTT client connected")
|
||||||
# background loop, which handles message publishing asynchronously.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
mqttc = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2)
|
|
||||||
mqttc.connect(MQTT_BROKER, MQTT_PORT)
|
|
||||||
mqttc.loop_start()
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# Publish immediately on reception — topic: {gateway_id}/{mac}/update
|
||||||
# CSV initialisation
|
# Only include fields that have been received at least once
|
||||||
# The header row is written only if the file does not already exist,
|
def publish(self, mac: str, data: dict):
|
||||||
# so that existing data is preserved across restarts.
|
topic = f"{self.gateway_id}/{mac}/update"
|
||||||
# ---------------------------------------------------------------------------
|
payload = {"timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")}
|
||||||
if not os.path.exists(CSV_FILE):
|
if data.get("temp") is not None:
|
||||||
with open(CSV_FILE, "w", newline="") as f:
|
payload["temp"] = data.get("temp")
|
||||||
writer = csv.writer(f)
|
if data.get("humidity") is not None:
|
||||||
writer.writerow(["timestamp", "room_id", "node_id", "temperature_c", "humidity_pct", "co2_ppm"])
|
payload["humidity"] = data.get("humidity")
|
||||||
|
if data.get("co2") is not None:
|
||||||
|
payload["co2_ppm"] = data.get("co2")
|
||||||
|
self.mqttc.publish(topic, json.dumps(payload))
|
||||||
|
log.info(f"Published to {topic} : {payload}")
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# BLE handlers — called by bleak on each notification
|
||||||
# BLE notification handlers
|
# Temp : 2 bytes (integer part + decimal part)
|
||||||
# These functions are called automatically by bleak each time the Thingy:52
|
# CO2 : uint16 little-endian, 0 means sensor is warming up
|
||||||
# sends a new value for the corresponding characteristic.
|
# Humidity: 1 byte, direct percentage value
|
||||||
#
|
def handle_temp(self, mac: str, sender, data: bytearray):
|
||||||
# Temperature encoding: 2 bytes
|
temp = data[0] + data[1] / 100
|
||||||
# byte 0 = integer part
|
if mac in self.latest:
|
||||||
# byte 1 = decimal part (hundredths)
|
self.latest[mac]["temp"] = round(temp, 2)
|
||||||
#
|
self.publish(mac, self.latest[mac])
|
||||||
# Humidity encoding: 1 byte
|
|
||||||
# byte 0 = relative humidity in percent
|
|
||||||
#
|
|
||||||
# CO2 encoding: 4 bytes (little-endian)
|
|
||||||
# bytes 0-1 = eCO2 in ppm (estimated CO2 derived from VOC measurement)
|
|
||||||
# bytes 2-3 = TVOC in ppb (not used here)
|
|
||||||
# A value of 0 indicates the sensor is still warming up.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
def handle_temp(mac, sender, data):
|
|
||||||
latest[mac]["temp"] = data[0] + data[1] / 100
|
|
||||||
|
|
||||||
def handle_humidity(mac, sender, data):
|
def handle_humidity(self, mac: str, sender, data: bytearray):
|
||||||
latest[mac]["humidity"] = data[0]
|
if mac in self.latest:
|
||||||
|
self.latest[mac]["humidity"] = data[0]
|
||||||
|
self.publish(mac, self.latest[mac])
|
||||||
|
|
||||||
def handle_co2(mac, sender, data):
|
def handle_co2(self, mac: str, sender, data: bytearray):
|
||||||
co2 = int.from_bytes(data[0:2], byteorder='little')
|
co2 = int.from_bytes(data[0:2], byteorder='little')
|
||||||
if co2 > 0:
|
if co2 == 0:
|
||||||
latest[mac]["co2"] = co2
|
log.warning(f"{mac} | CO2 sensor still warming up")
|
||||||
|
return
|
||||||
|
if mac in self.latest:
|
||||||
|
self.latest[mac]["co2"] = co2
|
||||||
|
self.publish(mac, self.latest[mac])
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# Stop scanner before connecting — BlueZ does not support both simultaneously.
|
||||||
# Data persistence
|
# Scanner is restarted once the device is connected and notifications registered.
|
||||||
# Each measurement is appended to both the CSV and JSON files.
|
async def connect_device(self, mac: str):
|
||||||
# The JSON file stores a list of all records, which is reloaded and
|
log.info(f"Attempting to connect to {mac}...")
|
||||||
# extended on each write to preserve the full history.
|
try:
|
||||||
# ---------------------------------------------------------------------------
|
if self.scanner:
|
||||||
def save_data(payload):
|
await self.scanner.stop()
|
||||||
with open(CSV_FILE, "a", newline="") as f:
|
await asyncio.sleep(1)
|
||||||
writer = csv.writer(f)
|
log.info(f"Connecting to {mac}...")
|
||||||
writer.writerow([
|
|
||||||
payload["timestamp"],
|
def on_disconnect(client):
|
||||||
payload["room_id"],
|
log.warning(f"{mac} disconnected, removing from active nodes")
|
||||||
payload["node_id"],
|
self.latest.pop(mac, None)
|
||||||
payload["sensors"]["temperature_c"],
|
|
||||||
payload["sensors"]["humidity_pct"],
|
client = BleakClient(mac, disconnected_callback=on_disconnect)
|
||||||
payload["sensors"]["co2_ppm"]
|
|
||||||
])
|
|
||||||
records = []
|
|
||||||
if os.path.exists(JSON_FILE):
|
|
||||||
with open(JSON_FILE, "r") as f:
|
|
||||||
try:
|
try:
|
||||||
records = json.load(f)
|
await asyncio.wait_for(client.connect(), timeout=10.0)
|
||||||
except:
|
except asyncio.TimeoutError:
|
||||||
records = []
|
log.error(f"Connection timeout for {mac}")
|
||||||
records.append(payload)
|
self.connecting.discard(mac)
|
||||||
with open(JSON_FILE, "w") as f:
|
self.latest.pop(mac, None)
|
||||||
json.dump(records, f, indent=2)
|
if self.scanner:
|
||||||
|
await self.scanner.start()
|
||||||
|
return
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
self.connecting.discard(mac)
|
||||||
# BLE device connection
|
self.latest[mac] = {"temp": None, "humidity": None, "co2": None}
|
||||||
# The scanner is stopped before connecting to avoid BLE stack conflicts on
|
log.info(f"{mac} connected")
|
||||||
# the Raspberry Pi. Once the connection is established and notifications are
|
|
||||||
# registered, the scanner is restarted to detect additional nodes.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
async def connect_device(mac):
|
|
||||||
global scanner
|
|
||||||
try:
|
|
||||||
if scanner:
|
|
||||||
await scanner.stop()
|
|
||||||
await asyncio.sleep(1)
|
|
||||||
|
|
||||||
client = BleakClient(mac)
|
await client.start_notify(self.uuid_temp,
|
||||||
await client.connect()
|
lambda s, d: self.handle_temp(mac, s, d))
|
||||||
|
await client.start_notify(self.uuid_co2,
|
||||||
|
lambda s, d: self.handle_co2(mac, s, d))
|
||||||
|
await client.start_notify(self.uuid_humidity,
|
||||||
|
lambda s, d: self.handle_humidity(mac, s, d))
|
||||||
|
|
||||||
node_id = discovered[mac]
|
await asyncio.sleep(1)
|
||||||
connecting.discard(mac)
|
await self.scanner.start()
|
||||||
print(f"{node_id} ({mac}) connected")
|
log.info("Scanner restarted, waiting for additional nodes")
|
||||||
|
|
||||||
await client.start_notify(UUID_TEMP, lambda s, d: handle_temp(mac, s, d))
|
except Exception as e:
|
||||||
await client.start_notify(UUID_CO2, lambda s, d: handle_co2(mac, s, d))
|
log.error(f"Connection error for {mac}: {e}")
|
||||||
await client.start_notify(UUID_HUMIDITY, lambda s, d: handle_humidity(mac, s, d))
|
self.connecting.discard(mac)
|
||||||
|
self.latest.pop(mac, None)
|
||||||
|
if self.scanner:
|
||||||
|
await self.scanner.start()
|
||||||
|
|
||||||
await asyncio.sleep(1)
|
# BLE discovery callback — filters on Thingy:52 service UUID
|
||||||
await scanner.start()
|
def on_device_found(self, device, adv_data):
|
||||||
print("Scanner restarted, waiting for additional nodes...")
|
uuids = [str(u).lower() for u in adv_data.service_uuids]
|
||||||
|
if self.service_uuid.lower() in uuids \
|
||||||
|
and device.address not in self.latest \
|
||||||
|
and device.address not in self.connecting:
|
||||||
|
self.connecting.add(device.address)
|
||||||
|
log.info(f"New node detected: {device.address}")
|
||||||
|
asyncio.ensure_future(self.connect_device(device.address))
|
||||||
|
|
||||||
except Exception as e:
|
# Start BLE scan and run indefinitely
|
||||||
print(f"Connection error for {mac} : {e}")
|
async def run(self):
|
||||||
connecting.discard(mac)
|
log.info("BLE scan started, waiting for Thingy:52 nodes...")
|
||||||
discovered.pop(mac, None)
|
self.scanner = BleakScanner(detection_callback=self.on_device_found)
|
||||||
latest.pop(mac, None)
|
await self.scanner.start()
|
||||||
if scanner:
|
while True:
|
||||||
await scanner.start()
|
await asyncio.sleep(1)
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# BLE discovery callback
|
|
||||||
# Called by bleak for every advertising packet received during the scan.
|
|
||||||
# A device is accepted only if it advertises the Thingy:52 environment
|
|
||||||
# service UUID, and only if it has not already been discovered or is not
|
|
||||||
# currently being connected to.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
def on_device_found(device, adv_data):
|
|
||||||
uuids = [str(u).lower() for u in adv_data.service_uuids]
|
|
||||||
if THINGY_SERVICE_UUID.lower() in uuids \
|
|
||||||
and device.address not in discovered \
|
|
||||||
and device.address not in connecting:
|
|
||||||
connecting.add(device.address)
|
|
||||||
thingy_counter[0] += 1
|
|
||||||
node_id = f"{ROOM_ID}_thingy{thingy_counter[0]}"
|
|
||||||
discovered[device.address] = node_id
|
|
||||||
latest[device.address] = {"temp": None, "humidity": None, "co2": None}
|
|
||||||
print(f"New node detected, assigned name: {node_id} ({device.address})")
|
|
||||||
asyncio.get_event_loop().create_task(connect_device(device.address))
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
if __name__ == "__main__":
|
||||||
# Periodic MQTT publishing
|
config_path = os.path.join(os.path.dirname(__file__), "config.json")
|
||||||
# Every INTERVAL seconds, the latest sensor values from all connected nodes
|
with open(config_path, "r") as f:
|
||||||
# are formatted as a JSON payload and published to the MQTT broker.
|
config = json.load(f)
|
||||||
# The topic structure follows: classroom/{room_id}/{node_id}
|
|
||||||
# Incomplete readings (sensor still warming up) are skipped for that cycle.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
async def publish_every_interval():
|
|
||||||
while True:
|
|
||||||
await asyncio.sleep(INTERVAL)
|
|
||||||
now = datetime.now(timezone.utc).isoformat()
|
|
||||||
print(f"\n--- {datetime.now().strftime('%H:%M:%S')} ---")
|
|
||||||
for mac, values in latest.items():
|
|
||||||
node_id = discovered.get(mac, mac)
|
|
||||||
if any(v is None for v in values.values()):
|
|
||||||
print(f"{node_id} | incomplete data, waiting for sensor warmup...")
|
|
||||||
continue
|
|
||||||
payload = {
|
|
||||||
"timestamp": now,
|
|
||||||
"room_id": ROOM_ID,
|
|
||||||
"node_id": node_id,
|
|
||||||
"sensors": {
|
|
||||||
"co2_ppm": values["co2"],
|
|
||||||
"temperature_c": round(values["temp"], 2),
|
|
||||||
"humidity_pct": values["humidity"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
topic = f"classroom/{ROOM_ID}/{node_id}"
|
|
||||||
mqttc.publish(topic, json.dumps(payload))
|
|
||||||
save_data(payload)
|
|
||||||
print(f"{node_id} | Temp: {values['temp']:.2f} C | Humidity: {values['humidity']}% | CO2: {values['co2']} ppm | Published")
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
gateway = Gateway(config)
|
||||||
# Entry point
|
asyncio.run(gateway.run())
|
||||||
# Starts the BLE scanner and runs the publishing loop indefinitely.
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
async def main():
|
|
||||||
global scanner
|
|
||||||
print("BLE scan started, waiting for Thingy:52 nodes...\n")
|
|
||||||
scanner = BleakScanner(detection_callback=on_device_found)
|
|
||||||
await scanner.start()
|
|
||||||
await publish_every_interval()
|
|
||||||
|
|
||||||
asyncio.run(main())
|
|
||||||
|
|||||||
Reference in New Issue
Block a user