You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
619 lines
19 KiB
619 lines
19 KiB
# =============================================================================
|
|
# Kynsight — Circular Buffer Kit (Improved Readability)
|
|
#
|
|
# Author: Kerem Yollu (improved with Claude)
|
|
# License: MIT
|
|
# Company: Kynsight
|
|
# Created: 2025-10-24
|
|
#
|
|
# WHAT IS A CIRCULAR BUFFER?
|
|
# ===========================
|
|
# Think of it like a tape recorder with limited tape that loops back:
|
|
#
|
|
# Write Position (wraps around)
|
|
# ↓
|
|
# [D][E][F][A][B][C] ← Buffer (capacity = 6 bytes)
|
|
# ↑ ↑
|
|
# oldest newest
|
|
#
|
|
# When full and you write more → oldest data gets overwritten (drop-oldest)
|
|
#
|
|
# KEY FEATURE: Absolute Write Offset (w_abs)
|
|
# ==========================================
|
|
# Unlike typical circular buffers that only track position (0-5 above),
|
|
# we ALSO track total bytes ever written:
|
|
#
|
|
# w_abs = 0: [] (empty)
|
|
# w_abs = 3: [A][B][C][ ][ ][ ] (3 bytes written)
|
|
# w_abs = 6: [A][B][C][D][E][F] (6 bytes, full)
|
|
# w_abs = 9: [G][H][I][D][E][F] (9 bytes total, overwrote A,B,C)
|
|
# ↑ newest ↑ oldest
|
|
#
|
|
# WHY? To copy data spans unambiguously:
|
|
# - "Give me bytes from w_abs=5 to w_abs=9" → [F][G][H][I]
|
|
# - Works even if buffer wrapped around multiple times
|
|
#
|
|
# C EQUIVALENT:
|
|
# =============
|
|
# In C, you'd use:
|
|
# - uint8_t* buffer (malloc'd array)
|
|
# - size_t write_pos = w_abs % capacity (for indexing)
|
|
# - pthread_mutex_t lock (for thread safety)
|
|
#
|
|
# In Python:
|
|
# - bytearray (like uint8_t* but managed memory)
|
|
# - % operator works same as C
|
|
# - threading.Lock (like pthread_mutex_t)
|
|
#
|
|
# =============================================================================
|
|
|
|
from __future__ import annotations
|
|
from dataclasses import dataclass, field
|
|
from enum import Enum
|
|
from typing import Tuple, Optional
|
|
import threading
|
|
|
|
|
|
# =============================================================================
|
|
# STATUS CODES (like errno in C)
|
|
# =============================================================================
|
|
|
|
class Status(Enum):
|
|
"""
|
|
Return codes for all operations.
|
|
|
|
C equivalent:
|
|
enum Status { OK=0, BAD_CONFIG=1, NOT_READY=2, OVERFLOW=3 };
|
|
|
|
Python advantage: Enum names are more readable than raw ints
|
|
"""
|
|
OK = 0 # Operation succeeded
|
|
BAD_CONFIG = 1 # Invalid parameter (e.g., size <= 0)
|
|
NOT_READY = 2 # Buffer not initialized
|
|
OVERFLOW = 3 # Data larger than capacity (was truncated)
|
|
|
|
|
|
# =============================================================================
|
|
# BUFFER HANDLE (like a struct in C)
|
|
# =============================================================================
|
|
|
|
@dataclass
|
|
class CircularBufferHandle:
|
|
"""
|
|
Opaque handle for the circular buffer state.
|
|
|
|
C equivalent:
|
|
typedef struct {
|
|
size_t capacity;
|
|
uint8_t* buffer;
|
|
size_t write_absolute;
|
|
int overflow_count;
|
|
pthread_mutex_t lock;
|
|
} CircularBuffer;
|
|
|
|
Python @dataclass: Automatically generates __init__, __repr__, etc.
|
|
Like a C struct but with methods and automatic constructor.
|
|
|
|
Fields explained:
|
|
-----------------
|
|
capacity:
|
|
Total size in bytes (like malloc(capacity) in C)
|
|
|
|
buffer:
|
|
Actual storage (bytearray = Python's managed uint8_t array)
|
|
|
|
write_absolute:
|
|
Total bytes written since creation (monotonic, never wraps)
|
|
To get position in buffer: write_absolute % capacity
|
|
|
|
overflow_count:
|
|
How many times we wrote more data than capacity at once
|
|
(these writes get truncated to keep newest data)
|
|
|
|
lock:
|
|
For thread safety (like pthread_mutex_t in C)
|
|
'with h.lock:' in Python = pthread_mutex_lock/unlock in C
|
|
"""
|
|
|
|
capacity: int
|
|
buffer: bytearray = field(default_factory=bytearray)
|
|
write_absolute: int = 0
|
|
overflow_count: int = 0
|
|
lock: threading.Lock = field(default_factory=threading.Lock)
|
|
|
|
|
|
# =============================================================================
|
|
# INITIALIZATION / CLEANUP
|
|
# =============================================================================
|
|
|
|
def cb_init(capacity_bytes: int) -> Tuple[Status, Optional[CircularBufferHandle]]:
|
|
"""
|
|
Create and initialize a circular buffer.
|
|
|
|
C equivalent:
|
|
CircularBuffer* cb_init(size_t capacity) {
|
|
if (capacity <= 0) return NULL;
|
|
CircularBuffer* cb = malloc(sizeof(CircularBuffer));
|
|
cb->buffer = malloc(capacity);
|
|
cb->capacity = capacity;
|
|
cb->write_absolute = 0;
|
|
cb->overflow_count = 0;
|
|
pthread_mutex_init(&cb->lock, NULL);
|
|
return cb;
|
|
}
|
|
|
|
Args:
|
|
capacity_bytes: Size of buffer in bytes (must be > 0)
|
|
|
|
Returns:
|
|
(Status.OK, handle) if successful
|
|
(Status.BAD_CONFIG, None) if capacity_bytes <= 0
|
|
|
|
Example:
|
|
status, buffer = cb_init(1024) # 1KB buffer
|
|
if status == Status.OK:
|
|
# use buffer...
|
|
"""
|
|
# Validation
|
|
if capacity_bytes <= 0:
|
|
return (Status.BAD_CONFIG, None)
|
|
|
|
# Create handle (like malloc + init in C)
|
|
handle = CircularBufferHandle(
|
|
capacity=capacity_bytes,
|
|
buffer=bytearray(capacity_bytes) # Pre-allocated, zero-filled
|
|
)
|
|
|
|
return (Status.OK, handle)
|
|
|
|
|
|
def cb_deinit(handle: CircularBufferHandle) -> Status:
|
|
"""
|
|
Clean up buffer resources.
|
|
|
|
C equivalent:
|
|
void cb_deinit(CircularBuffer* cb) {
|
|
free(cb->buffer);
|
|
pthread_mutex_destroy(&cb->lock);
|
|
free(cb);
|
|
}
|
|
|
|
Python note: Not strictly needed (garbage collector handles cleanup),
|
|
but included for API consistency and explicit resource management.
|
|
|
|
Args:
|
|
handle: Buffer to clean up
|
|
|
|
Returns:
|
|
Status.OK always
|
|
"""
|
|
# In Python, just returning is enough (GC will clean up)
|
|
# But we could explicitly clear if needed:
|
|
# handle.buffer = bytearray()
|
|
return Status.OK
|
|
|
|
|
|
def cb_reset(handle: CircularBufferHandle) -> Status:
|
|
"""
|
|
Reset buffer to empty state (zero bytes written).
|
|
|
|
Like flushing or clearing the buffer.
|
|
|
|
C equivalent:
|
|
void cb_reset(CircularBuffer* cb) {
|
|
pthread_mutex_lock(&cb->lock);
|
|
memset(cb->buffer, 0, cb->capacity);
|
|
cb->write_absolute = 0;
|
|
cb->overflow_count = 0;
|
|
pthread_mutex_unlock(&cb->lock);
|
|
}
|
|
|
|
Effects:
|
|
- Zeros all bytes in buffer
|
|
- Resets write_absolute to 0
|
|
- Resets overflow_count to 0
|
|
|
|
Thread-safe: Uses internal lock
|
|
|
|
Args:
|
|
handle: Buffer to reset
|
|
|
|
Returns:
|
|
Status.OK always
|
|
"""
|
|
with handle.lock: # ← Python's 'with' = pthread_mutex_lock/unlock in C
|
|
# Zero out the buffer
|
|
handle.buffer[:] = b"\x00" * handle.capacity
|
|
|
|
# Reset counters
|
|
handle.write_absolute = 0
|
|
handle.overflow_count = 0
|
|
|
|
return Status.OK
|
|
|
|
|
|
# =============================================================================
|
|
# WRITE OPERATION
|
|
# =============================================================================
|
|
|
|
def cb_write(handle: CircularBufferHandle, data: bytes) -> Tuple[Status, int]:
|
|
"""
|
|
Write bytes to the buffer (drop-oldest on overflow).
|
|
|
|
BEHAVIOR:
|
|
---------
|
|
If data fits: Write normally
|
|
If data > capacity: Keep only the NEWEST 'capacity' bytes
|
|
|
|
Example with capacity=4:
|
|
Initial: [ ][ ][ ][ ]
|
|
Write ABC: [A][B][C][ ] → w_abs=3
|
|
Write DE: [A][B][C][D] → w_abs=4 (full)
|
|
Write FG: [F][G][C][D] → w_abs=6 (overwrote A,B)
|
|
↑ newest ↑ oldest
|
|
|
|
C equivalent:
|
|
int cb_write(CircularBuffer* cb, const uint8_t* data, size_t len) {
|
|
pthread_mutex_lock(&cb->lock);
|
|
|
|
// Handle overflow
|
|
if (len > cb->capacity) {
|
|
data = data + (len - cb->capacity); // keep last N bytes
|
|
len = cb->capacity;
|
|
cb->overflow_count++;
|
|
}
|
|
|
|
// Calculate positions
|
|
size_t pos = cb->write_absolute % cb->capacity;
|
|
|
|
// Write (might wrap around)
|
|
if (pos + len <= cb->capacity) {
|
|
memcpy(&cb->buffer[pos], data, len);
|
|
} else {
|
|
size_t first_part = cb->capacity - pos;
|
|
memcpy(&cb->buffer[pos], data, first_part);
|
|
memcpy(&cb->buffer[0], data + first_part, len - first_part);
|
|
}
|
|
|
|
cb->write_absolute += len;
|
|
pthread_mutex_unlock(&cb->lock);
|
|
return len;
|
|
}
|
|
|
|
Args:
|
|
handle: Buffer to write to
|
|
data: Bytes to write
|
|
|
|
Returns:
|
|
(Status.OK, bytes_written)
|
|
where bytes_written = min(len(data), capacity)
|
|
|
|
Thread-safe: Yes
|
|
"""
|
|
# Edge case: empty write
|
|
if not data:
|
|
return (Status.OK, 0)
|
|
|
|
with handle.lock:
|
|
bytes_to_write = len(data)
|
|
|
|
# OVERFLOW HANDLING: If data > capacity, keep only newest bytes
|
|
if bytes_to_write > handle.capacity:
|
|
# Keep last 'capacity' bytes only
|
|
data = data[-handle.capacity:] # Python slice (negative index from end)
|
|
bytes_to_write = handle.capacity
|
|
handle.overflow_count += 1
|
|
|
|
# CALCULATE WRITE POSITION
|
|
# Like: size_t pos = write_absolute % capacity in C
|
|
position = handle.write_absolute % handle.capacity
|
|
end_position = position + bytes_to_write
|
|
|
|
# WRITE DATA (handle wrap-around)
|
|
if end_position <= handle.capacity:
|
|
# Case 1: Data fits without wrapping
|
|
# [___ABC___] → write at position 3
|
|
handle.buffer[position:end_position] = data
|
|
else:
|
|
# Case 2: Data wraps around to start
|
|
# [___ABC] → write "DEFGH" becomes [GH_DEF]
|
|
# ↑wrap here
|
|
|
|
# Split into two parts
|
|
first_part_size = handle.capacity - position # bytes until end
|
|
|
|
# Write first part (to end of buffer)
|
|
handle.buffer[position:] = data[:first_part_size]
|
|
|
|
# Write second part (wrapped to start)
|
|
handle.buffer[:end_position - handle.capacity] = data[first_part_size:]
|
|
|
|
# UPDATE ABSOLUTE POSITION (always increases, never wraps)
|
|
handle.write_absolute += bytes_to_write
|
|
|
|
return (Status.OK, bytes_to_write)
|
|
|
|
|
|
# =============================================================================
|
|
# READ OPERATION (SPAN COPY)
|
|
# =============================================================================
|
|
|
|
def cb_copy_span(
|
|
handle: CircularBufferHandle,
|
|
start_absolute: int,
|
|
end_absolute: int
|
|
) -> Tuple[Status, bytes]:
|
|
"""
|
|
Copy a span of bytes [start_absolute, end_absolute) from the buffer.
|
|
|
|
KEY CONCEPT: Uses absolute offsets, not ring positions!
|
|
|
|
Example with capacity=6, w_abs=10:
|
|
Buffer state: [G][H][I][J][D][E][F]
|
|
↑ newest (w_abs=10) ↑ oldest (w_abs=4)
|
|
|
|
Request: copy_span(5, 8) → want bytes at w_abs [5,6,7]
|
|
Result: [F][G][H]
|
|
|
|
If requested span > capacity:
|
|
Returns the newest 'capacity' bytes
|
|
(because older data was already overwritten)
|
|
|
|
C equivalent:
|
|
int cb_copy_span(CircularBuffer* cb, size_t start, size_t end,
|
|
uint8_t* out_buffer) {
|
|
pthread_mutex_lock(&cb->lock);
|
|
|
|
size_t len = end - start;
|
|
if (len > cb->capacity) {
|
|
// Can't retrieve more than capacity
|
|
start = cb->write_absolute - cb->capacity;
|
|
end = cb->write_absolute;
|
|
len = cb->capacity;
|
|
}
|
|
|
|
for (size_t i = 0; i < len; i++) {
|
|
size_t pos = (start + i) % cb->capacity;
|
|
out_buffer[i] = cb->buffer[pos];
|
|
}
|
|
|
|
pthread_mutex_unlock(&cb->lock);
|
|
return len;
|
|
}
|
|
|
|
Args:
|
|
handle: Buffer to read from
|
|
start_absolute: Start offset (absolute, not position in ring)
|
|
end_absolute: End offset (exclusive)
|
|
|
|
Returns:
|
|
(Status.OK, bytes_data)
|
|
|
|
Thread-safe: Yes
|
|
"""
|
|
# Edge case: empty span
|
|
if end_absolute <= start_absolute:
|
|
return (Status.OK, b"")
|
|
|
|
span_length = end_absolute - start_absolute
|
|
|
|
# OVERFLOW PROTECTION: Can't retrieve more than capacity
|
|
if span_length > handle.capacity:
|
|
# Adjust to newest available window
|
|
with handle.lock:
|
|
start_absolute = handle.write_absolute - handle.capacity
|
|
end_absolute = handle.write_absolute
|
|
span_length = handle.capacity
|
|
|
|
# ALLOCATE OUTPUT BUFFER
|
|
output = bytearray(span_length)
|
|
|
|
# COPY BYTES ONE BY ONE (handling wrap-around automatically)
|
|
with handle.lock:
|
|
for i in range(span_length):
|
|
# Calculate position in circular buffer
|
|
# Like: size_t pos = (start + i) % capacity in C
|
|
position = (start_absolute + i) % handle.capacity
|
|
output[i] = handle.buffer[position]
|
|
|
|
return (Status.OK, bytes(output))
|
|
|
|
|
|
# =============================================================================
|
|
# QUERY FUNCTIONS (read-only, thread-safe)
|
|
# =============================================================================
|
|
|
|
def cb_overflows(handle: CircularBufferHandle) -> int:
|
|
"""
|
|
Get number of overflow events.
|
|
|
|
An overflow happens when a single write is larger than capacity.
|
|
|
|
C equivalent:
|
|
int cb_overflows(CircularBuffer* cb) {
|
|
pthread_mutex_lock(&cb->lock);
|
|
int count = cb->overflow_count;
|
|
pthread_mutex_unlock(&cb->lock);
|
|
return count;
|
|
}
|
|
|
|
Returns:
|
|
Number of times data was truncated
|
|
"""
|
|
with handle.lock:
|
|
return handle.overflow_count
|
|
|
|
|
|
def cb_w_abs(handle: CircularBufferHandle) -> int:
|
|
"""
|
|
Get total bytes written since creation.
|
|
|
|
This is the absolute write offset (monotonic counter).
|
|
|
|
C equivalent:
|
|
size_t cb_w_abs(CircularBuffer* cb) {
|
|
pthread_mutex_lock(&cb->lock);
|
|
size_t w = cb->write_absolute;
|
|
pthread_mutex_unlock(&cb->lock);
|
|
return w;
|
|
}
|
|
|
|
Returns:
|
|
Total bytes written (never decreases, never wraps)
|
|
"""
|
|
with handle.lock:
|
|
return handle.write_absolute
|
|
|
|
|
|
def cb_capacity(handle: CircularBufferHandle) -> int:
|
|
"""
|
|
Get buffer capacity in bytes.
|
|
|
|
C equivalent:
|
|
size_t cb_capacity(CircularBuffer* cb) {
|
|
return cb->capacity; // No lock needed (const)
|
|
}
|
|
|
|
Returns:
|
|
Total capacity (fixed at creation)
|
|
"""
|
|
# No lock needed - capacity never changes
|
|
return handle.capacity
|
|
|
|
|
|
def cb_fill_bytes(handle: CircularBufferHandle) -> int:
|
|
"""
|
|
Get current fill level in bytes.
|
|
|
|
Definition:
|
|
fill = min(write_absolute, capacity)
|
|
|
|
Why min? Once we've written 'capacity' bytes, buffer is considered
|
|
full (even though we keep writing, overwriting old data).
|
|
|
|
Example with capacity=100:
|
|
w_abs=50 → fill=50 (half full)
|
|
w_abs=100 → fill=100 (full)
|
|
w_abs=200 → fill=100 (still "full", old data overwritten)
|
|
|
|
C equivalent:
|
|
size_t cb_fill_bytes(CircularBuffer* cb) {
|
|
pthread_mutex_lock(&cb->lock);
|
|
size_t fill = (cb->write_absolute < cb->capacity)
|
|
? cb->write_absolute
|
|
: cb->capacity;
|
|
pthread_mutex_unlock(&cb->lock);
|
|
return fill;
|
|
}
|
|
|
|
Returns:
|
|
Bytes currently in buffer (0 to capacity)
|
|
"""
|
|
with handle.lock:
|
|
# Ternary operator in Python: value_if_true if condition else value_if_false
|
|
# C equivalent: write_absolute < capacity ? write_absolute : capacity
|
|
return handle.write_absolute if handle.write_absolute < handle.capacity else handle.capacity
|
|
|
|
|
|
def cb_fill_pct(handle: CircularBufferHandle) -> int:
|
|
"""
|
|
Get fill level as percentage (0-100).
|
|
|
|
Useful for progress bars or monitoring.
|
|
|
|
C equivalent:
|
|
int cb_fill_pct(CircularBuffer* cb) {
|
|
size_t fill = cb_fill_bytes(cb);
|
|
size_t cap = cb->capacity > 0 ? cb->capacity : 1;
|
|
return (fill * 100) / cap; // Integer division
|
|
}
|
|
|
|
Returns:
|
|
Integer percentage 0-100 (no decimal, no '%' sign)
|
|
|
|
Example:
|
|
capacity=1000, fill=750 → returns 75
|
|
"""
|
|
fill = cb_fill_bytes(handle)
|
|
cap = handle.capacity if handle.capacity > 0 else 1 # Avoid division by zero
|
|
|
|
# Integer division (// in Python = / in C for integers)
|
|
return (fill * 100) // cap
|
|
|
|
|
|
# =============================================================================
|
|
# HELPER: Visual representation (for debugging)
|
|
# =============================================================================
|
|
|
|
def cb_debug_info(handle: CircularBufferHandle) -> str:
|
|
"""
|
|
Get human-readable debug information.
|
|
|
|
Not in original, but useful for learning/debugging!
|
|
|
|
Returns:
|
|
Multi-line string with buffer state
|
|
"""
|
|
with handle.lock:
|
|
info = f"""
|
|
Circular Buffer State:
|
|
----------------------
|
|
Capacity: {handle.capacity} bytes
|
|
Written total: {handle.write_absolute} bytes
|
|
Fill level: {cb_fill_bytes(handle)} bytes ({cb_fill_pct(handle)}%)
|
|
Overflows: {handle.overflow_count}
|
|
Position: {handle.write_absolute % handle.capacity}
|
|
"""
|
|
return info
|
|
|
|
|
|
# =============================================================================
|
|
# TEACHING NOTES
|
|
# =============================================================================
|
|
|
|
"""
|
|
KEY PYTHON vs C DIFFERENCES:
|
|
=============================
|
|
|
|
1. Memory Management:
|
|
C: malloc/free, manual cleanup
|
|
Python: Automatic garbage collection
|
|
|
|
2. Locking:
|
|
C: pthread_mutex_lock(&lock); ... pthread_mutex_unlock(&lock);
|
|
Python: with lock: ... (automatic unlock even if exception)
|
|
|
|
3. Arrays:
|
|
C: uint8_t* buffer (pointer arithmetic)
|
|
Python: bytearray (looks like list but is byte-optimized)
|
|
|
|
4. Slicing:
|
|
C: memcpy(dst, src, len)
|
|
Python: dst[:] = src (copies entire array)
|
|
dst[2:5] = src[2:5] (copies slice)
|
|
|
|
5. Modulo:
|
|
C: pos = offset % capacity;
|
|
Python: pos = offset % capacity (same!)
|
|
|
|
6. Enums:
|
|
C: enum Status { OK=0, ERROR=1 };
|
|
Python: class Status(Enum): OK=0; ERROR=1
|
|
|
|
7. Structs:
|
|
C: typedef struct { int x; } MyStruct;
|
|
Python: @dataclass class MyStruct: x: int
|
|
|
|
WHEN TO USE THIS BUFFER:
|
|
=========================
|
|
✅ Streaming data (serial, network)
|
|
✅ Multi-threaded producer/consumer
|
|
✅ Fixed memory budget
|
|
✅ Don't care about old data
|
|
✅ Need thread-safe operations
|
|
|
|
❌ Need to keep ALL data (use list/deque instead)
|
|
❌ Single-threaded (simpler structures work)
|
|
❌ Need to read from middle frequently (use deque)
|
|
"""
|