Raspberry Pi Serial Communication

import serial
import json
import time
import threading
import paho.mqtt.client as mqtt
from datetime import datetime

class SerialMQTTBridge:
    def __init__(self, serial_port, baudrate, mqtt_broker, mqtt_port=1883):
        # Serial setup
        self.serial = serial.Serial(serial_port, baudrate, timeout=1)
        
        # MQTT setup
        self.mqtt_client = mqtt.Client()
        self.mqtt_broker = mqtt_broker
        self.mqtt_port = mqtt_port
        
        # Topics
        self.data_topic = "rpi/serial/data"
        self.command_topic = "rpi/serial/command"
        self.status_topic = "rpi/serial/status"
        
        # State
        self.running = False
        self.device_id = f"rpi-{int(time.time())}"
        
    def setup_mqtt(self):
        """Configure MQTT client"""
        def on_connect(client, userdata, flags, rc):
            print(f"MQTT connected with code {rc}")
            client.subscribe(self.command_topic)
            
            # Publish online status
            status_msg = {
                'device_id': self.device_id,
                'status': 'online',
                'timestamp': datetime.now().isoformat()
            }
            client.publish(self.status_topic, json.dumps(status_msg))
            
        def on_message(client, userdata, msg):
            """Handle incoming MQTT commands"""
            try:
                command = json.loads(msg.payload.decode())
                self.handle_mqtt_command(command)
            except json.JSONDecodeError:
                # Treat as raw command
                self.serial.write(msg.payload + b'\n')
                
        self.mqtt_client.on_connect = on_connect
        self.mqtt_client.on_message = on_message
        
        # Set last will (offline status)
        offline_msg = {
            'device_id': self.device_id,
            'status': 'offline',
            'timestamp': datetime.now().isoformat()
        }
        self.mqtt_client.will_set(
            self.status_topic,
            json.dumps(offline_msg),
            retain=True
        )
        
    def handle_mqtt_command(self, command):
        """Process structured MQTT command"""
        cmd_type = command.get('type')
        
        if cmd_type == 'raw':
            # Send raw data to serial
            data = command.get('data', '')
            self.serial.write(data.encode() + b'\n')
            
        elif cmd_type == 'gpio':
            # GPIO control command
            pin = command.get('pin')
            action = command.get('action')
            value = command.get('value')
            
            gpio_command = f"GPIO {pin} {action} {value}\n"
            self.serial.write(gpio_command.encode())
            
        elif cmd_type == 'config':
            # Configuration command
            setting = command.get('setting')
            value = command.get('value')
            
            config_command = f"CONFIG {setting} {value}\n"
            self.serial.write(config_command.encode())
            
    def serial_to_mqtt_worker(self):
        """Forward serial data to MQTT"""
        buffer = b''
        
        while self.running:
            if self.serial.in_waiting:
                data = self.serial.read(self.serial.in_waiting)
                buffer += data
                
                # Process complete lines
                while b'\n' in buffer:
                    line, buffer = buffer.split(b'\n', 1)
                    line_str = line.decode('utf-8', errors='ignore').strip()
                    
                    if line_str:
                        # Create structured message
                        message = {
                            'device_id': self.device_id,
                            'timestamp': datetime.now().isoformat(),
                            'data': line_str,
                            'raw': True
                        }
                        
                        # Try to parse as JSON for structured data
                        try:
                            parsed_data = json.loads(line_str)
                            message['data'] = parsed_data
                            message['raw'] = False
                        except json.JSONDecodeError:
                            pass
                            
                        # Publish to MQTT
                        self.mqtt_client.publish(
                            self.data_topic,
                            json.dumps(message)
                        )
                        
            time.sleep(0.01)
            
    def start(self):
        """Start the bridge"""
        self.setup_mqtt()
        
        # Connect to MQTT broker
        self.mqtt_client.connect(self.mqtt_broker, self.mqtt_port, 60)
        self.mqtt_client.loop_start()
        
        # Start serial reader
        self.running = True
        self.serial_thread = threading.Thread(target=self.serial_to_mqtt_worker)
        self.serial_thread.daemon = True
        self.serial_thread.start()
        
        print(f"Serial-MQTT bridge started (Device ID: {self.device_id})")
        
    def stop(self):
        """Stop the bridge"""
        self.running = False
        
        # Publish offline status
        offline_msg = {
            'device_id': self.device_id,
            'status': 'offline',
            'timestamp': datetime.now().isoformat()
        }
        self.mqtt_client.publish(self.status_topic, json.dumps(offline_msg))
        
        # Cleanup
        if hasattr(self, 'serial_thread'):
            self.serial_thread.join()
            
        self.mqtt_client.loop_stop()
        self.mqtt_client.disconnect()
        self.serial.close()
        
        print("Bridge stopped")

# Usage
bridge = SerialMQTTBridge('/dev/serial0', 115200, 'localhost')
bridge.start()

try:
    # Keep bridge running
    while True:
        time.sleep(1)
except KeyboardInterrupt:
    bridge.stop()

import serial
import json
import threading
import time
from flask import Flask, request, jsonify
from datetime import datetime
from collections import deque

app = Flask(__name__)

class SerialAPIServer:
    def __init__(self, serial_port, baudrate):
        self.serial = serial.Serial(serial_port, baudrate, timeout=1)
        self.data_buffer = deque(maxlen=1000)  # Store last 1000 messages
        self.running = False
        self.lock = threading.Lock()
        
    def start_serial_reader(self):
        """Background thread to read serial data"""
        def reader():
            buffer = b''
            while self.running:
                if self.serial.in_waiting:
                    data = self.serial.read(self.serial.in_waiting)
                    buffer += data
                    
                    while b'\n' in buffer:
                        line, buffer = buffer.split(b'\n', 1)
                        line_str = line.decode('utf-8', errors='ignore').strip()
                        
                        if line_str:
                            with self.lock:
                                self.data_buffer.append({
                                    'timestamp': datetime.now().isoformat(),
                                    'data': line_str
                                })
                                
                time.sleep(0.01)
                
        self.running = True
        self.reader_thread = threading.Thread(target=reader)
        self.reader_thread.daemon = True
        self.reader_thread.start()
        
    def send_command(self, command):
        """Send command to serial device"""
        try:
            self.serial.write(command.encode() + b'\n')
            return True
        except:
            return False
            
    def get_recent_data(self, count=10):
        """Get recent data messages"""
        with self.lock:
            return list(self.data_buffer)[-count:]
            
    def stop(self):
        self.running = False
        if hasattr(self, 'reader_thread'):
            self.reader_thread.join()
        self.serial.close()

# Initialize serial API
serial_api = SerialAPIServer('/dev/serial0', 115200)
serial_api.start_serial_reader()

@app.route('/api/serial/send', methods=['POST'])
def send_serial_command():
    """Send command to serial device"""
    data = request.get_json()
    command = data.get('command', '')
    
    if serial_api.send_command(command):
        return jsonify({'status': 'success', 'command': command})
    else:
        return jsonify({'status': 'error', 'message': 'Failed to send command'}), 500

@app.route('/api/serial/data', methods=['GET'])
def get_serial_data():
    """Get recent serial data"""
    count = request.args.get('count', 10, type=int)
    data = serial_api.get_recent_data(count)
    return jsonify({'status': 'success', 'data': data})

@app.route('/api/serial/stream')
def stream_serial_data():
    """Server-sent events stream of serial data"""
    def generate():
        last_count = 0
        while True:
            current_data = serial_api.get_recent_data(1000)
            if len(current_data) > last_count:
                new_data = current_data[last_count:]
                for item in new_data:
                    yield f"data: {json.dumps(item)}\n\n"
                last_count = len(current_data)
            time.sleep(0.1)
            
    return app.response_class(
        generate(),
        mimetype='text/event-stream'
    )

@app.route('/api/system/info')
def system_info():
    """Get system information"""
    import psutil
    import os
    
    return jsonify({
        'raspberry_pi': {
            'model': open('/proc/device-tree/model', 'rb').read().decode().strip('\x00'),
            'serial_number': open('/proc/cpuinfo').read().split('Serial\t\t:')[-1].strip()[:16],
            'cpu_temp': float(open('/sys/class/thermal/thermal_zone0/temp').read()) / 1000,
            'uptime': open('/proc/uptime').read().split()[0]
        },
        'system': {
            'cpu_percent': psutil.cpu_percent(),
            'memory_percent': psutil.virtual_memory().percent,
            'disk_percent': psutil.disk_usage('/').percent
        }
    })

if __name__ == '__main__':
    try:
        app.run(host='0.0.0.0', port=5000, debug=False)
    finally:
        serial_api.stop()

import serial
import sqlite3
import json
import threading
import time
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime, timedelta

class SerialDataLogger:
    def __init__(self, serial_port, baudrate, db_path='serial_data.db'):
        self.serial = serial.Serial(serial_port, baudrate, timeout=1)
        self.db_path = db_path
        self.running = False
        
        # Initialize database
        self.init_database()
        
    def init_database(self):
        """Initialize SQLite database"""
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS serial_data (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                timestamp TEXT NOT NULL,
                raw_data TEXT NOT NULL,
                parsed_data TEXT,
                data_type TEXT,
                created_at DATETIME DEFAULT CURRENT_TIMESTAMP
            )
        ''')
        
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS sensor_readings (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                timestamp TEXT NOT NULL,
                sensor_type TEXT NOT NULL,
                value REAL NOT NULL,
                unit TEXT,
                created_at DATETIME DEFAULT CURRENT_TIMESTAMP
            )
        ''')
        
        conn.commit()
        conn.close()
        
    def parse_sensor_data(self, raw_data):
        """Parse common sensor data formats"""
        try:
            # Try JSON format
            data = json.loads(raw_data)
            return data, 'json'
        except json.JSONDecodeError:
            pass
            
        # Try CSV format: sensor_type,value,unit
        parts = raw_data.split(',')
        if len(parts) == 3:
            return {
                'sensor_type': parts[0].strip(),
                'value': float(parts[1].strip()),
                'unit': parts[2].strip()
            }, 'csv'
            
        # Try key-value format: temp=25.5C
        if '=' in raw_data:
            pairs = {}
            for pair in raw_data.split(','):
                if '=' in pair:
                    key, value = pair.split('=', 1)
                    pairs[key.strip()] = value.strip()
            return pairs, 'keyvalue'
            
        return None, 'unknown'
        
    def log_data(self, raw_data):
        """Log data to database"""
        timestamp = datetime.now().isoformat()
        parsed_data, data_type = self.parse_sensor_data(raw_data)
        
        conn = sqlite3.connect(self.db_path)
        cursor = conn.cursor()
        
        # Log raw data
        cursor.execute('''
            INSERT INTO serial_data (timestamp, raw_data, parsed_data, data_type)
            VALUES (?, ?, ?, ?)
        ''', (timestamp, raw_data, json.dumps(parsed_data) if parsed_data else None, data_type))
        
        # Log parsed sensor readings
        if parsed_data and isinstance(parsed_data, dict):
            if 'sensor_type' in parsed_data and 'value' in parsed_data:
                cursor.execute('''
                    INSERT INTO sensor_readings (timestamp, sensor_type, value, unit)
                    VALUES (?, ?, ?, ?)
                ''', (
                    timestamp,
                    parsed_data['sensor_type'],
                    parsed_data['value'],
                    parsed_data.get('unit', '')
                ))
            else:
                # Handle multiple sensors in one message
                for key, value in parsed_data.items():
                    try:
                        numeric_value = float(str(value).rstrip('C°F%'))
                        unit = str(value)[-1] if str(value)[-1] in 'C°F%' else ''
                        
                        cursor.execute('''
                            INSERT INTO sensor_readings (timestamp, sensor_type, value, unit)
                            VALUES (?, ?, ?, ?)
                        ''', (timestamp, key, numeric_value, unit))
                    except ValueError:
                        pass  # Skip non-numeric values
                        
        conn.commit()
        conn.close()
        
    def data_logger_worker(self):
        """Background worker for logging data"""
        buffer = b''
        
        while self.running:
            if self.serial.in_waiting:
                data = self.serial.read(self.serial.in_waiting)
                buffer += data
                
                while b'\n' in buffer:
                    line, buffer = buffer.split(b'\n', 1)
                    line_str = line.decode('utf-8', errors='ignore').strip()
                    
                    if line_str:
                        self.log_data(line_str)
                        print(f"Logged: {line_str}")
                        
            time.sleep(0.01)
            
    def start_logging(self):
        """Start data logging"""
        self.running = True
        self.logger_thread = threading.Thread(target=self.data_logger_worker)
        self.logger_thread.daemon = True
        self.logger_thread.start()
        print("Data logging started")
        
    def stop_logging(self):
        """Stop data logging"""
        self.running = False
        if hasattr(self, 'logger_thread'):
            self.logger_thread.join()
        self.serial.close()
        print("Data logging stopped")
        
    def generate_report(self, hours=24):
        """Generate analysis report"""
        end_time = datetime.now()
        start_time = end_time - timedelta(hours=hours)
        
        conn = sqlite3.connect(self.db_path)
        
        # Get sensor data
        df = pd.read_sql_query('''
            SELECT timestamp, sensor_type, value, unit
            FROM sensor_readings
            WHERE timestamp >= ? AND timestamp <= ?
            ORDER BY timestamp
        ''', conn, params=(start_time.isoformat(), end_time.isoformat()))
        
        if df.empty:
            print("No data found for the specified time range")
            return
            
        # Convert timestamp to datetime
        df['timestamp'] = pd.to_datetime(df['timestamp'])
        
        # Generate plots
        fig, axes = plt.subplots(2, 2, figsize=(15, 10))
        fig.suptitle(f'Sensor Data Analysis - Last {hours} Hours')
        
        # Plot 1: Temperature over time
        temp_data = df[df['sensor_type'] == 'temperature']
        if not temp_data.empty:
            axes[0, 0].plot(temp_data['timestamp'], temp_data['value'])
            axes[0, 0].set_title('Temperature Over Time')
            axes[0, 0].set_ylabel('Temperature (°C)')
            axes[0, 0].tick_params(axis='x', rotation=45)
            
        # Plot 2: Data distribution
        sensor_counts = df['sensor_type'].value_counts()
        axes[0, 1].pie(sensor_counts.values, labels=sensor_counts.index, autopct='%1.1f%%')
        axes[0, 1].set_title('Sensor Data Distribution')
        
        # Plot 3: Value ranges by sensor
        df.boxplot(column='value', by='sensor_type', ax=axes[1, 0])
        axes[1, 0].set_title('Value Ranges by Sensor Type')
        axes[1, 0].set_xlabel('Sensor Type')
        
        # Plot 4: Data frequency over time
        hourly_counts = df.set_index('timestamp').resample('H').size()
        axes[1, 1].bar(range(len(hourly_counts)), hourly_counts.values)
        axes[1, 1].set_title('Data Points Per Hour')
        axes[1, 1].set_ylabel('Count')
        
        plt.tight_layout()
        plt.savefig('sensor_report.png', dpi=300, bbox_inches='tight')
        plt.show()
        
        # Print statistics
        print("\n=== Data Summary ===")
        print(f"Total data points: {len(df)}")
        print(f"Unique sensors: {df['sensor_type'].nunique()}")
        print(f"Time range: {df['timestamp'].min()} to {df['timestamp'].max()}")
        
        print("\n=== Sensor Statistics ===")
        for sensor_type in df['sensor_type'].unique():
            sensor_data = df[df['sensor_type'] == sensor_type]
            print(f"\n{sensor_type.upper()}:")
            print(f"  Count: {len(sensor_data)}")
            print(f"  Min: {sensor_data['value'].min():.2f}")
            print(f"  Max: {sensor_data['value'].max():.2f}")
            print(f"  Mean: {sensor_data['value'].mean():.2f}")
            print(f"  Std: {sensor_data['value'].std():.2f}")
            
        conn.close()

# Usage example
logger = SerialDataLogger('/dev/serial0', 115200)
logger.start_logging()

try:
    # Let it run for a while
    time.sleep(60)  # Log for 1 minute
    
    # Generate report
    logger.generate_report(hours=1)
    
except KeyboardInterrupt:
    pass
finally:
    logger.stop_logging()