Practice and reinforce the concepts from Lesson 8
Build an advanced AR interaction system with natural gesture controls including tap-to-place, drag-to-move, pinch-to-scale, rotate gestures, and voice commands. Create an intuitive interface that makes AR feel natural and responsive.
An AR app with sophisticated gesture recognition that responds to single-touch, multi-touch, and voice commands. Objects will animate smoothly between states, provide haptic feedback, and show visual hints for available interactions.

Build a comprehensive gesture detection system:
// utils/GestureRecognizer.js
export class GestureRecognizer {
constructor() {
this.touches = [];
this.gestureState = 'idle';
this.thresholds = {
tap: 200, // ms
longPress: 500, // ms
dragDistance: 10, // pixels
pinchScale: 0.05, // sensitivity
};
this.listeners = new Map();
}
on(eventType, callback) {
if (!this.listeners.has(eventType)) {
this.listeners.set(eventType, []);
}
this.listeners.get(eventType).push(callback);
}
emit(eventType, data) {
const callbacks = this.listeners.get(eventType) || [];
callbacks.forEach(cb => cb(data));
}
handleTouchStart(event) {
const touches = Array.from(event.nativeEvent.touches).map(t => ({
id: t.identifier,
startX: t.pageX,
startY: t.pageY,
currentX: t.pageX,
currentY: t.pageY,
startTime: Date.now(),
}));
this.touches = touches;
if (touches.length === 1) {
this.startLongPressTimer(touches[0]);
} else if (touches.length === 2) {
this.gestureState = 'multitouch';
this.initialPinchDistance = this.getDistance(touches[0], touches[1]);
this.initialRotation = this.getAngle(touches[0], touches[1]);
}
}
handleTouchMove(event) {
if (this.touches.length === 0) return;
const touches = Array.from(event.nativeEvent.touches);
touches.forEach((touch, index) => {
if (this.touches[index]) {
this.touches[index].currentX = touch.pageX;
this.touches[index].currentY = touch.pageY;
}
});
if (this.touches.length === 1) {
const touch = this.touches[0];
const distance = this.getDistance(
{ currentX: touch.startX, currentY: touch.startY },
{ currentX: touch.currentX, currentY: touch.currentY }
);
if (distance > this.thresholds.dragDistance) {
this.cancelLongPressTimer();
this.gestureState = 'dragging';
this.emit('drag', {
x: touch.currentX,
y: touch.currentY,
deltaX: touch.currentX - touch.startX,
deltaY: touch.currentY - touch.startY,
});
}
} else if (this.touches.length === 2) {
this.handleMultiTouch();
}
}
handleTouchEnd(event) {
const remainingTouches = event.nativeEvent.touches.length;
if (remainingTouches === 0) {
if (this.gestureState === 'idle' && this.touches.length === 1) {
const touch = this.touches[0];
const duration = Date.now() - touch.startTime;
if (duration < this.thresholds.tap) {
this.emit('tap', {
x: touch.currentX,
y: touch.currentY,
});
}
}
this.cancelLongPressTimer();
this.gestureState = 'idle';
this.touches = [];
} else {
this.touches = Array.from(event.nativeEvent.touches).map(t => ({
id: t.identifier,
startX: t.pageX,
startY: t.pageY,
currentX: t.pageX,
currentY: t.pageY,
startTime: Date.now(),
}));
}
}
handleMultiTouch() {
if (this.touches.length !== 2) return;
const currentDistance = this.getDistance(
this.touches[0],
this.touches[1]
);
const currentRotation = this.getAngle(
this.touches[0],
this.touches[1]
);
// Pinch to scale
if (this.initialPinchDistance) {
const scale = currentDistance / this.initialPinchDistance;
this.emit('pinch', { scale });
}
// Rotate
if (this.initialRotation !== null) {
const rotation = currentRotation - this.initialRotation;
this.emit('rotate', { rotation });
}
}
startLongPressTimer(touch) {
this.longPressTimer = setTimeout(() => {
if (this.gestureState === 'idle') {
this.emit('longPress', {
x: touch.currentX,
y: touch.currentY,
});
}
}, this.thresholds.longPress);
}
cancelLongPressTimer() {
if (this.longPressTimer) {
clearTimeout(this.longPressTimer);
this.longPressTimer = null;
}
}
getDistance(touch1, touch2) {
const dx = touch2.currentX - touch1.currentX;
const dy = touch2.currentY - touch1.currentY;
return Math.sqrt(dx * dx + dy * dy);
}
getAngle(touch1, touch2) {
return Math.atan2(
touch2.currentY - touch1.currentY,
touch2.currentX - touch1.currentX
);
}
reset() {
this.touches = [];
this.gestureState = 'idle';
this.cancelLongPressTimer();
}
}
š” Tip: Haptic feedback makes gestures feel more responsive. Use
Haptics.impactAsync()when users interact with objects.
Create a wrapper for haptic responses:
// utils/HapticFeedback.js
import * as Haptics from 'expo-haptics';
export class HapticFeedback {
static async light() {
try {
await Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
} catch (error) {
// Haptics not supported
}
}
static async medium() {
try {
await Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium);
} catch (error) {
// Haptics not supported
}
}
static async heavy() {
try {
await Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Heavy);
} catch (error) {
// Haptics not supported
}
}
static async success() {
try {
await Haptics.notificationAsync(
Haptics.NotificationFeedbackType.Success
);
} catch (error) {
// Haptics not supported
}
}
static async error() {
try {
await Haptics.notificationAsync(
Haptics.NotificationFeedbackType.Error
);
} catch (error) {
// Haptics not supported
}
}
static async selection() {
try {
await Haptics.selectionAsync();
} catch (error) {
// Haptics not supported
}
}
}
Install the haptics package:
npm install expo-haptics
Build visual indicators for interactions:
// components/InteractionHints.js
import React, { useEffect, useRef } from 'react';
import { View, Text, StyleSheet, Animated } from 'react-native';
const InteractionHints = ({ visible, hint }) => {
const fadeAnim = useRef(new Animated.Value(0)).current;
const pulseAnim = useRef(new Animated.Value(1)).current;
useEffect(() => {
if (visible) {
Animated.parallel([
Animated.timing(fadeAnim, {
toValue: 1,
duration: 300,
useNativeDriver: true,
}),
Animated.loop(
Animated.sequence([
Animated.timing(pulseAnim, {
toValue: 1.1,
duration: 800,
useNativeDriver: true,
}),
Animated.timing(pulseAnim, {
toValue: 1,
duration: 800,
useNativeDriver: true,
}),
])
),
]).start();
} else {
Animated.timing(fadeAnim, {
toValue: 0,
duration: 300,
useNativeDriver: true,
}).start();
}
}, [visible]);
const hints = {
tap: {
icon: 'š',
text: 'Tap to select',
subtext: 'Long press for options',
},
drag: {
icon: 'ā',
text: 'Drag to move',
subtext: 'Slide across surface',
},
pinch: {
icon: 'š¤',
text: 'Pinch to resize',
subtext: 'Use two fingers',
},
rotate: {
icon: 'š',
text: 'Rotate to turn',
subtext: 'Twist with two fingers',
},
place: {
icon: 'š',
text: 'Tap to place',
subtext: 'Point at a surface',
},
};
const currentHint = hints[hint] || hints.tap;
return (
<Animated.View
style={[
styles.container,
{
opacity: fadeAnim,
transform: [{ scale: pulseAnim }],
},
]}
>
<Text style={styles.icon}>{currentHint.icon}</Text>
<Text style={styles.text}>{currentHint.text}</Text>
<Text style={styles.subtext}>{currentHint.subtext}</Text>
</Animated.View>
);
};
const styles = StyleSheet.create({
container: {
position: 'absolute',
bottom: 200,
left: '50%',
transform: [{ translateX: -100 }],
width: 200,
backgroundColor: 'rgba(0, 0, 0, 0.85)',
borderRadius: 15,
padding: 20,
alignItems: 'center',
},
icon: {
fontSize: 48,
marginBottom: 10,
},
text: {
color: 'white',
fontSize: 16,
fontWeight: 'bold',
textAlign: 'center',
marginBottom: 5,
},
subtext: {
color: '#aaaaaa',
fontSize: 12,
textAlign: 'center',
},
});
export default InteractionHints;
Add smooth transitions for object interactions:
// utils/ObjectAnimator.js
import * as THREE from 'three';
export class ObjectAnimator {
constructor() {
this.animations = new Map();
}
animateScale(object, targetScale, duration = 300) {
const startScale = object.scale.x;
const startTime = Date.now();
const animationId = `scale_${object.uuid}`;
this.animations.set(animationId, {
update: () => {
const elapsed = Date.now() - startTime;
const progress = Math.min(elapsed / duration, 1);
// Ease out cubic
const eased = 1 - Math.pow(1 - progress, 3);
const currentScale = startScale + (targetScale - startScale) * eased;
object.scale.setScalar(currentScale);
if (progress >= 1) {
this.animations.delete(animationId);
}
},
});
}
animatePosition(object, targetPosition, duration = 300) {
const startPosition = object.position.clone();
const startTime = Date.now();
const animationId = `position_${object.uuid}`;
this.animations.set(animationId, {
update: () => {
const elapsed = Date.now() - startTime;
const progress = Math.min(elapsed / duration, 1);
const eased = 1 - Math.pow(1 - progress, 3);
object.position.lerpVectors(startPosition, targetPosition, eased);
if (progress >= 1) {
this.animations.delete(animationId);
}
},
});
}
animateRotation(object, targetRotation, duration = 300) {
const startRotation = object.rotation.y;
const startTime = Date.now();
const animationId = `rotation_${object.uuid}`;
this.animations.set(animationId, {
update: () => {
const elapsed = Date.now() - startTime;
const progress = Math.min(elapsed / duration, 1);
const eased = 1 - Math.pow(1 - progress, 3);
object.rotation.y = startRotation + (targetRotation - startRotation) * eased;
if (progress >= 1) {
this.animations.delete(animationId);
}
},
});
}
pulse(object, scale = 1.2, duration = 200) {
const originalScale = object.scale.x;
const startTime = Date.now();
const animationId = `pulse_${object.uuid}`;
this.animations.set(animationId, {
update: () => {
const elapsed = Date.now() - startTime;
const progress = elapsed / duration;
if (progress < 0.5) {
// Scale up
const s = originalScale + (scale - originalScale) * (progress * 2);
object.scale.setScalar(s);
} else {
// Scale down
const s = scale - (scale - originalScale) * ((progress - 0.5) * 2);
object.scale.setScalar(s);
}
if (progress >= 1) {
object.scale.setScalar(originalScale);
this.animations.delete(animationId);
}
},
});
}
update() {
this.animations.forEach(animation => {
animation.update();
});
}
clear() {
this.animations.clear();
}
}
Build a radial menu for object actions:
// components/ContextMenu.js
import React, { useEffect, useRef } from 'react';
import {
View,
Text,
TouchableOpacity,
StyleSheet,
Animated,
} from 'react-native';
const ContextMenu = ({ visible, position, onAction, onClose }) => {
const scaleAnim = useRef(new Animated.Value(0)).current;
useEffect(() => {
if (visible) {
Animated.spring(scaleAnim, {
toValue: 1,
tension: 50,
friction: 7,
useNativeDriver: true,
}).start();
} else {
Animated.timing(scaleAnim, {
toValue: 0,
duration: 200,
useNativeDriver: true,
}).start();
}
}, [visible]);
if (!visible) return null;
const actions = [
{ id: 'duplicate', icon: 'š', label: 'Copy' },
{ id: 'delete', icon: 'šļø', label: 'Delete' },
{ id: 'rotate90', icon: 'ā»', label: 'Rotate 90°' },
{ id: 'resetSize', icon: 'āļø', label: 'Reset Size' },
];
const radius = 80;
const angleStep = (Math.PI * 2) / actions.length;
return (
<>
<TouchableOpacity
style={styles.overlay}
activeOpacity={1}
onPress={onClose}
/>
<View
style={[
styles.container,
{ left: position.x - 40, top: position.y - 40 },
]}
>
<Animated.View
style={[
styles.center,
{ transform: [{ scale: scaleAnim }] },
]}
>
<Text style={styles.centerIcon}>āļø</Text>
</Animated.View>
{actions.map((action, index) => {
const angle = angleStep * index - Math.PI / 2;
const x = Math.cos(angle) * radius;
const y = Math.sin(angle) * radius;
return (
<Animated.View
key={action.id}
style={[
styles.actionContainer,
{
transform: [
{ translateX: x },
{ translateY: y },
{ scale: scaleAnim },
],
},
]}
>
<TouchableOpacity
style={styles.action}
onPress={() => {
onAction(action.id);
onClose();
}}
>
<Text style={styles.actionIcon}>{action.icon}</Text>
</TouchableOpacity>
<Text style={styles.actionLabel}>{action.label}</Text>
</Animated.View>
);
})}
</View>
</>
);
};
const styles = StyleSheet.create({
overlay: {
...StyleSheet.absoluteFillObject,
backgroundColor: 'rgba(0, 0, 0, 0.3)',
},
container: {
position: 'absolute',
width: 80,
height: 80,
},
center: {
width: 80,
height: 80,
borderRadius: 40,
backgroundColor: 'white',
justifyContent: 'center',
alignItems: 'center',
shadowColor: '#000',
shadowOffset: { width: 0, height: 2 },
shadowOpacity: 0.3,
shadowRadius: 5,
elevation: 5,
},
centerIcon: {
fontSize: 32,
},
actionContainer: {
position: 'absolute',
left: 40,
top: 40,
width: 60,
height: 60,
marginLeft: -30,
marginTop: -30,
alignItems: 'center',
},
action: {
width: 60,
height: 60,
borderRadius: 30,
backgroundColor: '#4ECDC4',
justifyContent: 'center',
alignItems: 'center',
shadowColor: '#000',
shadowOffset: { width: 0, height: 2 },
shadowOpacity: 0.3,
shadowRadius: 3,
elevation: 5,
},
actionIcon: {
fontSize: 24,
},
actionLabel: {
marginTop: 5,
fontSize: 10,
fontWeight: 'bold',
color: 'white',
textAlign: 'center',
textShadowColor: 'rgba(0, 0, 0, 0.5)',
textShadowOffset: { width: 0, height: 1 },
textShadowRadius: 2,
},
});
export default ContextMenu;
Combine all gesture systems:
// App.js integration additions
import { GestureRecognizer } from './utils/GestureRecognizer';
import { HapticFeedback } from './utils/HapticFeedback';
import { ObjectAnimator } from './utils/ObjectAnimator';
import InteractionHints from './components/InteractionHints';
import ContextMenu from './components/ContextMenu';
// In component:
const [currentHint, setCurrentHint] = useState('place');
const [contextMenu, setContextMenu] = useState({ visible: false, position: { x: 0, y: 0 } });
const gestureRecognizer = useRef(new GestureRecognizer());
const objectAnimator = useRef(new ObjectAnimator());
useEffect(() => {
const recognizer = gestureRecognizer.current;
recognizer.on('tap', async (data) => {
await HapticFeedback.light();
handleTap(data.x, data.y);
});
recognizer.on('longPress', async (data) => {
await HapticFeedback.medium();
setContextMenu({
visible: true,
position: { x: data.x, y: data.y },
});
});
recognizer.on('drag', (data) => {
handleDrag(data.x, data.y);
setCurrentHint('drag');
});
recognizer.on('pinch', async (data) => {
handlePinch(data.scale);
setCurrentHint('pinch');
if (Math.abs(data.scale - 1) > 0.1) {
await HapticFeedback.selection();
}
});
recognizer.on('rotate', (data) => {
handleRotate(data.rotation);
setCurrentHint('rotate');
});
return () => {
recognizer.reset();
};
}, []);
// Add to render loop
useEffect(() => {
const animate = () => {
objectAnimator.current.update();
requestAnimationFrame(animate);
};
animate();
}, []);
Problem: Gestures conflict with each other Solution: Implement proper gesture state management. Cancel conflicting gestures when a new gesture starts. Use thresholds to distinguish between tap and drag.
Problem: Haptic feedback not working Solution: Verify expo-haptics is installed. Check device settings allow haptic feedback. Wrap haptic calls in try-catch as some devices don't support it.
Problem: Animations are choppy Solution: Use requestAnimationFrame for smooth 60fps updates. Avoid heavy calculations in animation loop. Consider reducing animation complexity on lower-end devices.
Problem: Context menu appears in wrong position Solution: Account for screen coordinates vs AR world coordinates. Adjust menu position based on screen edges to prevent clipping.
Problem: Multi-touch gestures feel unresponsive Solution: Reduce gesture recognition thresholds. Process touch events more frequently. Add visual feedback immediately when gesture starts.
For advanced students:
Voice Commands: Add voice recognition to place/move/delete objects hands-free using expo-speech-recognition
Gesture Shortcuts: Implement custom gestures like swipe-up to duplicate or circle gesture to group objects
Undo/Redo: Add gesture-based undo (three-finger swipe left) and redo (three-finger swipe right)
Smart Snapping: Objects snap to grid, edges, or other objects when moved close
Tutorial Mode: Record and playback gesture demonstrations for new users
In this activity, you:
In the next lesson, you'll explore Health Connect API to build fitness tracking features. You'll learn how to read health data, track workouts, and display wellness statistics.