A plushie that talks back, recorded
Built for
the prototype that became Luno
Project
Plushie voice toy
The toy compresses audio to IMA ADPCM to fit the microcontroller's memory, the server decodes it by hand, transcribes it, generates a reply, and sends speech back. The recording below is a real exchange from the July 2025 device test.
Demo
Incoming call from Child with the plushie
Live signal
awaiting first turn…
- turn
- 0/4
- agent replies
- 0
- audio clips
- 0/2
Press play to hear the real recording.
Real recording from 31 July 2025, captured by the Flask backend during a device test. The text is a description of each clip, not a transcription.
How it works
01
IMA ADPCM at 4 bits per sample halves the upload from a memory-constrained ESP32.
02
Hand-written decoder on the server mirrors the firmware encoder exactly.
03
Plain HTTP upload and download: simplest thing that worked on day one.
04
Every exchange is saved as WAV, which is why a real recording exists.
Stack and code
- ESP32
- Arduino C++
- Flask
- OpenAI Whisper
- GPT
- TTS
- AWS EC2
app.py— Flask backend with the ADPCM decoder.
# ESP32 Toy Backend 123
from flask import Flask, request, send_file, jsonify
import os, datetime, time
import struct
import wave
from whisper_stt import transcribe_audio
from gpt_reply import get_gpt_reply
from tts import synthesize_speech
app = Flask(__name__)
TEMP_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "temp")
AUDIO_DIR = os.path.join(os.path.dirname(__file__), "audio")
os.makedirs(TEMP_DIR, exist_ok=True)
os.makedirs(AUDIO_DIR, exist_ok=True)
def decompress_adpcm_to_wav(adpcm_data, output_path, sample_rate=16000):
"""
Decompress ADPCM data to WAV format
Assumes IMA ADPCM format commonly used by ESP32
"""
try:
# IMA ADPCM step size table
step_table = [
7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
]
# Index table for IMA ADPCM (matches ESP32 implementation)
index_table = [-1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8]
# Initialize decoder state
predicted_sample = 0
step_index = 0
decoded_samples = []
# Process ADPCM data
for byte in adpcm_data:
# Each byte contains two 4-bit ADPCM samples
for nibble in [(byte & 0x0F), (byte >> 4)]:
step = step_table[step_index]
# Decode the nibble (matching ESP32 encoder logic)
diffq = step >> 3
if nibble & 4:
diffq += step
if nibble & 2:
diffq += step >> 1
if nibble & 1:
diffq += step >> 2
if nibble & 8:
predicted_sample -= diffq
else:
predicted_sample += diffq
# Clamp to 16-bit range
predicted_sample = max(-32768, min(32767, predicted_sample))
decoded_samples.append(predicted_sample)
# Update step index
step_index += index_table[nibble]
step_index = max(0, min(88, step_index))
~/Documents/Plushie Server Code/app.py
plushiev3_wifi_audio_recording.ino— ESP32 firmware: record, compress, upload.
/*
* PlushieAI ESP32-S3 WiFi Manager & Backend Connection with Audio Playback + Voice Recording
*
* Required Libraries (install via Arduino Library Manager or PlatformIO):
* - ESPAsyncWebServer: https://github.com/me-no-dev/ESPAsyncWebServer
* - AsyncTCP: https://github.com/me-no-dev/AsyncTCP
* - DNSServer: Built-in with ESP32 core
* - Preferences: Built-in with ESP32 core
* - HTTPClient: Built-in with ESP32 core
* - WiFi: Built-in with ESP32 core
* - FFat: Built-in with ESP32 core
* - I2S: Built-in with ESP32 core
*
* Features:
* - WiFi management with captive portal
* - Audio playback on backend connection
* - Press-to-record voice capture with PSRAM buffering
* - ADPCM compression and HTTP upload
* -
*/
#include <WiFi.h>
#include <ESPAsyncWebServer.h>
#include <DNSServer.h>
#include <Preferences.h>
#include <HTTPClient.h>
#include <FFat.h>
#include <driver/i2s.h>
#include <vector>
#include <esp_heap_caps.h>
#include <esp_psram.h>
#define USE_ADPCM_COMPRESSION true
// Compile-time check for ESP32-S3
#if !CONFIG_IDF_TARGET_ESP32S3
#error "This code is designed specifically for ESP32-S3"
#endif
// Hardware pin definitions
#define BUTTON_PIN 21
#define MIC_POWER_PIN 2
// I2S ports
#define MIC_I2S_PORT I2S_NUM_0
#define SPK_I2S_PORT I2S_NUM_1
// Microphone pins
#define I2S_MIC_WS 15
#define I2S_MIC_SCK 18
#define I2S_MIC_SD 6
// Speaker pins
#define I2S_SPK_WS 4
#define I2S_SPK_SCK 5
#define I2S_SPK_DIN 17
// Constants
#define LED_BUILTIN RGB_BUILTIN
#define WIFI_TIMEOUT 10000 // 10 seconds
#define SERVER_RETRY_DELAY 5000 // 5 seconds
#define AP_NAME "PlushieAI-Setup"
#define BACKEND_HOST "http://18.190.135.144"
#define WAKE_FILE_PATH "/wake.wav"
#define MIN_FILE_SIZE 1024 // 1KB minimum file size
// Voice recording constants
#define SAMPLE_RATE 16000
#define BITS_PER_SAMPLE 16
#define DEBOUNCE_TIME_MS 50
#define INITIAL_BUFFER_SIZE (1024 * 512) // 512KB initial
#define BUFFER_GROW_SIZE (1024 * 256) // 256KB growth chunks
#define MAX_PSRAM_USAGE 0.9f // 90% of available PSRAM
#define RECORDING_GAIN 2.5f // Volume boost multiplier
#define SOFT_LIMIT_THRESH 0.5f // Threshold for soft limiting (50% of max)
#define LOW_PASS_ALPHA 0.15f // Low-pass filter coefficient (0.1-0.3, lower = more filtering)
// Audio compression constants
#ifdef USE_ADPCM_COMPRESSION
#define ADPCM_BLOCK_SIZE 256 // Process in 256 sample blocks
#else
~/Documents/Arduino/plushiev3_wifi_audio_recording/plushiev3_wifi_audio_recording.ino