All projects
Voice agentJuly to August 2025Measured

A plushie that talks back, recorded

Built for

the prototype that became Luno

Project

Plushie voice toy

The toy compresses audio to IMA ADPCM to fit the microcontroller's memory, the server decodes it by hand, transcribes it, generates a reply, and sends speech back. The recording below is a real exchange from the July 2025 device test.

02

Demo

Plushiestandby

Incoming call from Child with the plushie

0 / 4

Live signal

awaiting first turn…

turn
0/4
agent replies
0
audio clips
0/2

Press play to hear the real recording.

Real recording from 31 July 2025, captured by the Flask backend during a device test. The text is a description of each clip, not a transcription.

03

How it works

Architecture diagram
  1. 01

    IMA ADPCM at 4 bits per sample halves the upload from a memory-constrained ESP32.

  2. 02

    Hand-written decoder on the server mirrors the firmware encoder exactly.

  3. 03

    Plain HTTP upload and download: simplest thing that worked on day one.

  4. 04

    Every exchange is saved as WAV, which is why a real recording exists.

05

Stack and code

app.py
# ESP32 Toy Backend 123
from flask import Flask, request, send_file, jsonify
import os, datetime, time
import struct
import wave

from whisper_stt import transcribe_audio
from gpt_reply import get_gpt_reply
from tts import synthesize_speech

app = Flask(__name__)
TEMP_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "temp")
AUDIO_DIR = os.path.join(os.path.dirname(__file__), "audio")
os.makedirs(TEMP_DIR, exist_ok=True)
os.makedirs(AUDIO_DIR, exist_ok=True)

def decompress_adpcm_to_wav(adpcm_data, output_path, sample_rate=16000):
    """
    Decompress ADPCM data to WAV format
    Assumes IMA ADPCM format commonly used by ESP32
    """
    try:
        # IMA ADPCM step size table
        step_table = [
            7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
            19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
            50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
            130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
            337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
            876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
            2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
            5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
            15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
        ]
        
        # Index table for IMA ADPCM (matches ESP32 implementation)
        index_table = [-1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8]
        
        # Initialize decoder state
        predicted_sample = 0
        step_index = 0
        decoded_samples = []
        
        # Process ADPCM data
        for byte in adpcm_data:
            # Each byte contains two 4-bit ADPCM samples
            for nibble in [(byte & 0x0F), (byte >> 4)]:
                step = step_table[step_index]
                
                # Decode the nibble (matching ESP32 encoder logic)
                diffq = step >> 3
                if nibble & 4:
                    diffq += step
                if nibble & 2:
                    diffq += step >> 1
                if nibble & 1:
                    diffq += step >> 2
                
                if nibble & 8:
                    predicted_sample -= diffq
                else:
                    predicted_sample += diffq
                
                # Clamp to 16-bit range
                predicted_sample = max(-32768, min(32767, predicted_sample))
                decoded_samples.append(predicted_sample)
                
                # Update step index
                step_index += index_table[nibble]
                step_index = max(0, min(88, step_index))

~/Documents/Plushie Server Code/app.py

plushiev3_wifi_audio_recording.ino
/*
 * PlushieAI ESP32-S3 WiFi Manager & Backend Connection with Audio Playback + Voice Recording
 * 
 * Required Libraries (install via Arduino Library Manager or PlatformIO):
 * - ESPAsyncWebServer: https://github.com/me-no-dev/ESPAsyncWebServer
 * - AsyncTCP: https://github.com/me-no-dev/AsyncTCP
 * - DNSServer: Built-in with ESP32 core
 * - Preferences: Built-in with ESP32 core
 * - HTTPClient: Built-in with ESP32 core
 * - WiFi: Built-in with ESP32 core
 * - FFat: Built-in with ESP32 core
 * - I2S: Built-in with ESP32 core
 * 
 * Features:
 * - WiFi management with captive portal
 * - Audio playback on backend connection
 * - Press-to-record voice capture with PSRAM buffering
 * - ADPCM compression and HTTP upload
 * - 
 */

#include <WiFi.h>
#include <ESPAsyncWebServer.h>
#include <DNSServer.h>
#include <Preferences.h>
#include <HTTPClient.h>
#include <FFat.h>
#include <driver/i2s.h>
#include <vector>
#include <esp_heap_caps.h>
#include <esp_psram.h>
#define USE_ADPCM_COMPRESSION true

// Compile-time check for ESP32-S3
#if !CONFIG_IDF_TARGET_ESP32S3
#error "This code is designed specifically for ESP32-S3"
#endif

// Hardware pin definitions
#define BUTTON_PIN          21
#define MIC_POWER_PIN       2

// I2S ports
#define MIC_I2S_PORT        I2S_NUM_0
#define SPK_I2S_PORT        I2S_NUM_1

// Microphone pins
#define I2S_MIC_WS          15
#define I2S_MIC_SCK         18
#define I2S_MIC_SD          6

// Speaker pins
#define I2S_SPK_WS          4
#define I2S_SPK_SCK         5
#define I2S_SPK_DIN         17

// Constants
#define LED_BUILTIN         RGB_BUILTIN
#define WIFI_TIMEOUT        10000  // 10 seconds
#define SERVER_RETRY_DELAY  5000   // 5 seconds
#define AP_NAME             "PlushieAI-Setup"
#define BACKEND_HOST        "http://18.190.135.144"
#define WAKE_FILE_PATH      "/wake.wav"
#define MIN_FILE_SIZE       1024   // 1KB minimum file size

// Voice recording constants
#define SAMPLE_RATE         16000
#define BITS_PER_SAMPLE     16
#define DEBOUNCE_TIME_MS    50
#define INITIAL_BUFFER_SIZE (1024 * 512)  // 512KB initial
#define BUFFER_GROW_SIZE    (1024 * 256)  // 256KB growth chunks
#define MAX_PSRAM_USAGE     0.9f          // 90% of available PSRAM
#define RECORDING_GAIN      2.5f          // Volume boost multiplier
#define SOFT_LIMIT_THRESH   0.5f          // Threshold for soft limiting (50% of max)
#define LOW_PASS_ALPHA      0.15f         // Low-pass filter coefficient (0.1-0.3, lower = more filtering)

// Audio compression constants
#ifdef USE_ADPCM_COMPRESSION
#define ADPCM_BLOCK_SIZE    256           // Process in 256 sample blocks
#else

~/Documents/Arduino/plushiev3_wifi_audio_recording/plushiev3_wifi_audio_recording.ino