From 17697ba6ec992216d250c242dcb41f7e21e2b721 Mon Sep 17 00:00:00 2001
From: syuilo <4439005+syuilo@users.noreply.github.com>
Date: Sun, 26 Apr 2026 17:08:51 +0900
Subject: [PATCH] joystick

---
 packages/frontend/src/pages/room.vue          | 106 +++++++++++++++++-
 packages/frontend/src/world/joystick.ts       |  61 ++++++++++
 .../frontend/src/world/room/controller.ts     |  21 +++-
 packages/frontend/src/world/room/engine.ts    |  75 ++++++++-----
 packages/frontend/src/world/utility.ts        |  49 ++++++++
 5 files changed, 276 insertions(+), 36 deletions(-)
 create mode 100644 packages/frontend/src/world/joystick.ts

diff --git a/packages/frontend/src/pages/room.vue b/packages/frontend/src/pages/room.vue
index f1f6d14e68..007665e224 100644
--- a/packages/frontend/src/pages/room.vue
+++ b/packages/frontend/src/pages/room.vue
@@ -19,7 +19,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 			</div>
 		</Transition>
 
-		<template v-if="!isZenMode">
+		<div :class="$style.overlayBottom">
 			<div v-if="controller.isReady.value" class="_buttonsCenter" :class="$style.overlayControls">
 				<template v-if="controller.isEditMode.value">
 					<MkButton v-if="controller.grabbing.value" @click="cancelGrabbing"><i class="ti ti-x"></i> cancel</MkButton>
@@ -39,6 +39,19 @@ SPDX-License-Identifier: AGPL-3.0-only
 				</template>
 			</div>
 
+			<div v-if="useVirtualJoystick" :class="$style.joySticks">
+				<div ref="joyStickLeftEl" :class="$style.joyStickLeft" :style="{ '--startXPx': (joyStickLeftStartPos?.x ?? 0) + 'px', '--startYPx': (joyStickLeftStartPos?.y ?? 0) + 'px', '--rPx': joyStickRadiusPx + 'px' }">
+					<div v-show="joyStickLeftStartPos != null" :class="$style.joyStickRangeCircle"></div>
+					<div v-show="joyStickLeftVec.x !== 0 || joyStickLeftVec.y !== 0" :class="$style.joyStickPuck" :style="{ '--x': joyStickLeftVec.x, '--y': joyStickLeftVec.y }"></div>
+				</div>
+				<div ref="joyStickRightEl" :class="$style.joyStickRight" :style="{ '--startXPx': (joyStickRightStartPos?.x ?? 0) + 'px', '--startYPx': (joyStickRightStartPos?.y ?? 0) + 'px', '--rPx': joyStickRadiusPx + 'px' }">
+					<div v-show="joyStickRightStartPos != null" :class="$style.joyStickRangeCircle"></div>
+					<div v-show="joyStickRightVec.x !== 0 || joyStickRightVec.y !== 0" :class="$style.joyStickPuck" :style="{ '--x': joyStickRightVec.x, '--y': joyStickRightVec.y }"></div>
+				</div>
+			</div>
+		</div>
+
+		<template v-if="!isZenMode">
 			<div v-if="controller.isReady.value && controller.isEditMode.value && controller.selected.value != null && !controller.grabbing.value" :key="controller.selected.value.objectId" class="_panel" :class="$style.overlayObjectInfoPanel">
 				{{ controller.selected.value.objectDef.name }}
 
@@ -172,6 +185,8 @@ import { deepClone } from '@/utility/clone.js';
 import { GRAPHICS_QUALITY_HIGH, GRAPHICS_QUALITY_LOW, GRAPHICS_QUALITY_MEDIUM } from '@/world/room/engine.js';
 import { deviceKind } from '@/utility/device-kind.js';
 import MkProgressBar from '@/components/MkProgressBar.vue';
+import { Joystick } from '@/world/joystick.js';
+import { isTouchUsing } from '@/utility/touch.js';
 
 const canvas = useTemplateRef('canvas');
 
@@ -190,6 +205,15 @@ const isZenMode = ref(false);
 const isRoomSettingsOpen = ref(false);
 const isChanged = ref(false);
 const graphicsQuality = ref<number>(deviceKind === 'smartphone' ? GRAPHICS_QUALITY_LOW : GRAPHICS_QUALITY_MEDIUM);
+const useVirtualJoystick = isTouchUsing && (deviceKind === 'smartphone' || deviceKind === 'tablet');
+
+const joyStickRadiusPx = 100;
+const joyStickLeftEl = useTemplateRef('joyStickLeftEl');
+const joyStickRightEl = useTemplateRef('joyStickRightEl');
+const joyStickLeftVec = ref({ x: 0, y: 0 });
+const joyStickRightVec = ref({ x: 0, y: 0 });
+const joyStickLeftStartPos = ref<{ x: number; y: number } | null>(null);
+const joyStickRightStartPos = ref<{ x: number; y: number } | null>(null);
 
 const data = localStorage.getItem('roomData') != null ? JSON.parse(localStorage.getItem('roomData')!) : {
 	heya: {
@@ -230,6 +254,7 @@ let latestData = deepClone(data);
 
 const controller = new RoomController(data, {
 	graphicsQuality: graphicsQuality.value,
+	useVirtualJoystick,
 });
 
 onMounted(async () => {
@@ -255,6 +280,32 @@ onMounted(async () => {
 	//		}));
 	//	}
 	//});
+
+	if (joyStickLeftEl.value != null && joyStickRightEl.value != null) {
+		const joyStickLeft = new Joystick(joyStickLeftEl.value!, { radiusPx: joyStickRadiusPx });
+		joyStickLeft.on('start', (vector) => {
+			joyStickLeftStartPos.value = vector;
+		});
+		joyStickLeft.on('end', () => {
+			joyStickLeftStartPos.value = null;
+		});
+		joyStickLeft.on('updateVector', (vector) => {
+			joyStickLeftVec.value = vector;
+			controller.setCameraJoystickMoveVector(vector);
+		});
+
+		const joyStickRight = new Joystick(joyStickRightEl.value!, { radiusPx: joyStickRadiusPx });
+		joyStickRight.on('start', (vector) => {
+			joyStickRightStartPos.value = vector;
+		});
+		joyStickRight.on('end', () => {
+			joyStickRightStartPos.value = null;
+		});
+		joyStickRight.on('updateVector', (vector) => {
+			joyStickRightVec.value = vector;
+			controller.setCameraJoystickRotateVector(vector);
+		});
+	}
 });
 
 onUnmounted(() => {
@@ -362,6 +413,7 @@ async function revert() {
 async function reload() {
 	await controller.reset(null, {
 		graphicsQuality: graphicsQuality.value,
+		useVirtualJoystick,
 	});
 }
 
@@ -426,10 +478,54 @@ definePage(() => ({
 	}
 }
 
-.controls {
+.joySticks {
+	display: flex;
+	width: 100%;
 }
 
-.overlayControls {
+.joyStickLeft, .joyStickRight {
+	position: relative;
+	flex: 1;
+	height: 70px;
+	box-sizing: border-box;
+	padding: 8px;
+}
+
+.joyStickLeft::before, .joyStickRight::before {
+	content: '';
+	display: block;
+	width: 100%;
+	height: 100%;
+	border: solid 2px #fff;
+	border-radius: 16px;
+	pointer-events: none;
+}
+
+.joyStickRangeCircle {
+	position: absolute;
+	top: var(--startYPx);
+	left: var(--startXPx);
+	width: calc(var(--rPx) * 2);
+	height: calc(var(--rPx) * 2);
+	border: solid 2px rgba(255, 255, 255, 0.5);
+	border-radius: 100%;
+	transform: translate(-50%, -50%);
+	pointer-events: none;
+}
+
+.joyStickPuck {
+	position: absolute;
+	top: calc(var(--startYPx) + (var(--y) * var(--rPx)));
+	left: calc(var(--startXPx) + (var(--x) * var(--rPx)));
+	width: 30px;
+	height: 30px;
+	background: #fff;
+	border-radius: 100%;
+	transform: translate(-50%, -50%);
+	pointer-events: none;
+}
+
+.overlayBottom {
 	position: absolute;
 	bottom: 0;
 	left: 0;
@@ -437,6 +533,10 @@ definePage(() => ({
 	width: 100%;
 }
 
+.overlayControls {
+
+}
+
 .overlayObjectInfoPanel {
 	position: absolute;
 	top: 16px;
diff --git a/packages/frontend/src/world/joystick.ts b/packages/frontend/src/world/joystick.ts
new file mode 100644
index 0000000000..b5f8ec6100
--- /dev/null
+++ b/packages/frontend/src/world/joystick.ts
@@ -0,0 +1,61 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { EventEmitter } from 'eventemitter3';
+
+export class Joystick extends EventEmitter<{
+	'start': (vec: { x: number; y: number }) => void;
+	'end': () => void;
+	'updateVector': (vec: { x: number; y: number }) => void; // -1.0 ~ 1.0
+}> {
+	private el: HTMLDivElement;
+	private startPos: { x: number; y: number } | null = null;
+	private radiusPx: number;
+
+	constructor(el: HTMLDivElement, options: { radiusPx: number }) {
+		super();
+		this.el = el;
+		this.radiusPx = options.radiusPx;
+		this.el.addEventListener('pointerdown', this.onPointerDown);
+		this.el.addEventListener('pointermove', this.onPointerMove);
+		this.el.addEventListener('pointerup', this.onPointerUp);
+		this.el.addEventListener('touchstart', ev => ev.preventDefault(), { passive: false });
+		this.el.addEventListener('touchmove', ev => ev.preventDefault(), { passive: false });
+	}
+
+	private onPointerDown = (ev: PointerEvent) => {
+		ev.preventDefault();
+		this.el.setPointerCapture(ev.pointerId);
+		this.startPos = { x: ev.offsetX, y: ev.offsetY };
+		this.emit('start', this.startPos);
+	};
+
+	private onPointerMove = (ev: PointerEvent) => {
+		ev.preventDefault();
+		if (this.startPos == null) return;
+		const vec = {
+			x: ev.offsetX - this.startPos.x,
+			y: ev.offsetY - this.startPos.y,
+		};
+		const len = Math.sqrt(vec.x * vec.x + vec.y * vec.y);
+		if (len > this.radiusPx) {
+			vec.x = (vec.x / len) * this.radiusPx;
+			vec.y = (vec.y / len) * this.radiusPx;
+		}
+		const normVec = {
+			x: vec.x / this.radiusPx,
+			y: vec.y / this.radiusPx,
+		};
+		this.emit('updateVector', normVec);
+	};
+
+	private onPointerUp = (ev: PointerEvent) => {
+		ev.preventDefault();
+		this.el.releasePointerCapture(ev.pointerId);
+		this.startPos = null;
+		this.emit('end');
+		this.emit('updateVector', { x: 0, y: 0 });
+	};
+}
diff --git a/packages/frontend/src/world/room/controller.ts b/packages/frontend/src/world/room/controller.ts
index b8486adaf2..312d706a7b 100644
--- a/packages/frontend/src/world/room/controller.ts
+++ b/packages/frontend/src/world/room/controller.ts
@@ -16,6 +16,7 @@ import * as sound from '@/utility/sound.js';
 type Options = {
 	workerMode?: boolean;
 	graphicsQuality: number;
+	useVirtualJoystick?: boolean;
 };
 
 // 抽象化レイヤー
@@ -59,7 +60,7 @@ export class RoomController {
 		if (this.options.workerMode) {
 			const offscreen = canvas.transferControlToOffscreen();
 			this.worker = new RoomWorker();
-			this.worker.postMessage({ type: 'init', canvas: offscreen, roomState: this.roomState.value, graphicsQuality: this.options.graphicsQuality }, [offscreen]);
+			this.worker.postMessage({ type: 'init', canvas: offscreen, roomState: this.roomState.value, graphicsQuality: this.options.graphicsQuality, useVirtualJoystick: this.options.useVirtualJoystick }, [offscreen]);
 			this.isReady.value = true;
 		} else {
 			const babylonEngine = new BABYLON.WebGPUEngine(canvas, { doNotHandleContextLost: true });
@@ -67,7 +68,7 @@ export class RoomController {
 			babylonEngine.enableOfflineSupport = false;
 			await babylonEngine.initAsync();
 
-			this.engine = new RoomEngine(this.roomState.value, { canvas, engine: babylonEngine, graphicsQuality: this.options.graphicsQuality });
+			this.engine = new RoomEngine(this.roomState.value, { canvas, engine: babylonEngine, graphicsQuality: this.options.graphicsQuality, useVirtualJoystick: this.options.useVirtualJoystick });
 			this.engine.on('loadingProgress', ({ progress }) => {
 				this.initializeProgress.value = progress;
 			});
@@ -201,6 +202,22 @@ export class RoomController {
 		}
 	}
 
+	public setCameraJoystickMoveVector(vec: { x: number; y: number }) {
+		if (this.worker != null) {
+			this.worker.postMessage({ type: 'setCameraJoystickMoveVector', vec });
+		} else if (this.engine != null) {
+			this.engine.cameraJoystickMove(vec);
+		}
+	}
+
+	public setCameraJoystickRotateVector(vec: { x: number; y: number }) {
+		if (this.worker != null) {
+			this.worker.postMessage({ type: 'setCameraJoystickRotateVector', vec });
+		} else if (this.engine != null) {
+			this.engine.cameraJoystickRotate(vec);
+		}
+	}
+
 	public enterEditMode() {
 		if (this.worker != null) {
 			this.worker.postMessage({ type: 'enterEditMode' });
diff --git a/packages/frontend/src/world/room/engine.ts b/packages/frontend/src/world/room/engine.ts
index 23cfedabf9..e9f009dab9 100644
--- a/packages/frontend/src/world/room/engine.ts
+++ b/packages/frontend/src/world/room/engine.ts
@@ -15,7 +15,7 @@ import { registerBuiltInLoaders } from '@babylonjs/loaders/dynamic';
 import { BoundingBoxRenderer } from '@babylonjs/core/Rendering/boundingBoxRenderer';
 import { GridMaterial } from '@babylonjs/materials';
 import { EventEmitter } from 'eventemitter3';
-import { TIME_MAP, scaleMorph, camelToKebab, cm, WORLD_SCALE, getMeshesBoundingBox, Timer, getYRotationDirection } from '../utility.js';
+import { TIME_MAP, scaleMorph, camelToKebab, cm, WORLD_SCALE, getMeshesBoundingBox, Timer, getYRotationDirection, FreeCameraTouchVirtualJoystickInput } from '../utility.js';
 import { getObjectDef } from './object-defs.js';
 import { findMaterial, ModelManager, SYSTEM_HEYA_MESH_NAMES, SYSTEM_MESH_NAMES } from './utility.js';
 import { SimpleHeyaManager } from './heya.js';
@@ -23,8 +23,6 @@ import type { HeyaManager, JapaneseHeyaOptions, SimpleHeyaOptions } from './heya
 import type { ObjectDef, RoomObjectInstance, RoomStateObject } from './object.js';
 import { genId } from '@/utility/id.js';
 import { deepClone } from '@/utility/clone.js';
-import { isTouchUsing } from '@/utility/touch.js';
-import { deviceKind } from '@/utility/device-kind.js';
 
 const BAKE_TRANSFORM = false; // 実験的
 const SNAPSHOT_RENDERING = true; // 実験的
@@ -226,6 +224,7 @@ export class RoomEngine extends EventEmitter<RoomEngineEvents> {
 		canvas: HTMLCanvasElement;
 		engine: BABYLON.WebGPUEngine;
 		graphicsQuality: number;
+		useVirtualJoystick?: boolean;
 	}) {
 		super();
 
@@ -289,14 +288,7 @@ export class RoomEngine extends EventEmitter<RoomEngineEvents> {
 
 		this.scene.collisionsEnabled = true;
 
-		const useVirtualJoystick = isTouchUsing && (deviceKind === 'smartphone' || deviceKind === 'tablet');
-
-		this.camera = useVirtualJoystick ? new BABYLON.VirtualJoysticksCamera('camera', new BABYLON.Vector3(0, cm(130), cm(0)), this.scene) : new BABYLON.UniversalCamera('camera', new BABYLON.Vector3(0, cm(130), cm(0)), this.scene);
-		this.camera.attachControl(this.canvas);
-		if (useVirtualJoystick) {
-			(this.camera.inputs.attached.virtualJoystick as BABYLON.FreeCameraVirtualJoystickInput).getLeftJoystick().setJoystickSensibility(0.3);
-			(this.camera.inputs.attached.virtualJoystick as BABYLON.FreeCameraVirtualJoystickInput).getRightJoystick().setJoystickSensibility(0.025);
-		}
+		this.camera = options.useVirtualJoystick ? new BABYLON.FreeCamera('camera', new BABYLON.Vector3(0, cm(130), cm(0)), this.scene) : new BABYLON.UniversalCamera('camera', new BABYLON.Vector3(0, cm(130), cm(0)), this.scene);
 		this.camera.minZ = cm(1);
 		this.camera.maxZ = RENDER_OUTDOOR_ENV ? cm(10000) : cm(1000);
 		this.camera.fov = 1;
@@ -304,26 +296,39 @@ export class RoomEngine extends EventEmitter<RoomEngineEvents> {
 		this.camera.checkCollisions = true;
 		this.camera.applyGravity = true;
 		this.camera.needMoveForGravity = true;
-		this.camera.keysUp.push(87); // W
-		this.camera.keysDown.push(83); // S
-		this.camera.keysLeft.push(65); // A
-		this.camera.keysRight.push(68); // D
-		const normalSpeed = 0.02 * WORLD_SCALE;
-		this.camera.speed = normalSpeed;
-		this.scene.onKeyboardObservable.add((kbInfo) => {
-			switch (kbInfo.type) {
-				case BABYLON.KeyboardEventTypes.KEYDOWN:
-					if (kbInfo.event.key === 'Shift') {
-						this.camera.speed = normalSpeed * 4;
-					}
-					break;
-				case BABYLON.KeyboardEventTypes.KEYUP:
-					if (kbInfo.event.key === 'Shift') {
-						this.camera.speed = normalSpeed;
-					}
-					break;
-			}
-		});
+
+		if (options.useVirtualJoystick) {
+			this.camera.inputs.clear();
+			this.camera.inputs.add(new FreeCameraTouchVirtualJoystickInput({
+				moveSensitivity: 0.015 * WORLD_SCALE,
+				rotationSensitivity: 0.1,
+			}));
+			this.camera.inertia = 0.75;
+		} else {
+			const normalSpeed = 0.02 * WORLD_SCALE;
+			this.camera.speed = normalSpeed;
+
+			this.camera.keysUp.push(87); // W
+			this.camera.keysDown.push(83); // S
+			this.camera.keysLeft.push(65); // A
+			this.camera.keysRight.push(68); // D
+			this.scene.onKeyboardObservable.add((kbInfo) => {
+				switch (kbInfo.type) {
+					case BABYLON.KeyboardEventTypes.KEYDOWN:
+						if (kbInfo.event.key === 'Shift') {
+							this.camera.speed = normalSpeed * 4;
+						}
+						break;
+					case BABYLON.KeyboardEventTypes.KEYUP:
+						if (kbInfo.event.key === 'Shift') {
+							this.camera.speed = normalSpeed;
+						}
+						break;
+				}
+			});
+		}
+
+		this.camera.attachControl(this.canvas);
 
 		//this.scene.activeCamera = this.camera;
 
@@ -448,6 +453,14 @@ export class RoomEngine extends EventEmitter<RoomEngineEvents> {
 		}
 	}
 
+	public cameraJoystickMove(vector: { x: number; y: number; }) {
+		this.camera.inputs.attached.joystick.setJoystickMoveVector(vector);
+	}
+
+	public cameraJoystickRotate(vector: { x: number; y: number; }) {
+		this.camera.inputs.attached.joystick.setJoystickRotationVector(vector);
+	}
+
 	public async init() {
 		await this.loadHeya();
 		if (RENDER_OUTDOOR_ENV) await this.loadEnvModel();
diff --git a/packages/frontend/src/world/utility.ts b/packages/frontend/src/world/utility.ts
index 62b14b0ecd..b3ce1657aa 100644
--- a/packages/frontend/src/world/utility.ts
+++ b/packages/frontend/src/world/utility.ts
@@ -607,3 +607,52 @@ export function getRgb(hex: string | number): [number, number, number] | null {
 	if (m == null) return [0, 0, 0];
 	return m.map(x => parseInt(x, 16) / 255) as [number, number, number];
 }
+
+export class FreeCameraTouchVirtualJoystickInput implements BABYLON.ICameraInput<BABYLON.FreeCamera> {
+	public camera: BABYLON.FreeCamera;
+	private joystickMoveSensitivity: number;
+	private joystickRotationSensitivity: number;
+	private joystickMoveVector = BABYLON.Vector3.Zero();
+	private joystickRotationVecX = 0;
+	private joystickRotationVecY = 0;
+
+	constructor(options: {
+		moveSensitivity?: number;
+		rotationSensitivity?: number;
+	}) {
+		this.joystickMoveSensitivity = options.moveSensitivity ?? 0.01;
+		this.joystickRotationSensitivity = options.rotationSensitivity ?? 0.01;
+	}
+
+	getClassName = () => this.constructor.name;
+
+	getSimpleName = () => 'joystick';
+
+	attachControl(noPreventDefault) {
+	}
+
+	detachControl() {
+	}
+
+	public setJoystickMoveVector(vec: { x: number; y: number }) {
+		this.joystickMoveVector = new BABYLON.Vector3(vec.x, 0, -vec.y).scale(this.joystickMoveSensitivity);
+	}
+
+	public setJoystickRotationVector(vec: { x: number; y: number }) {
+		let directionAdjust = 1;
+		if (this.camera.getScene().useRightHandedSystem) directionAdjust *= -1;
+		if (this.camera.parent && this.camera.parent._getWorldMatrixDeterminant() < 0) directionAdjust *= -1;
+
+		this.joystickRotationVecX = vec.y * this.joystickRotationSensitivity * this.joystickRotationSensitivity;
+		this.joystickRotationVecY = vec.x * this.joystickRotationSensitivity * directionAdjust * this.joystickRotationSensitivity;
+	}
+
+	checkInputs() {
+		this.camera.cameraRotation.y += this.joystickRotationVecY;
+		this.camera.cameraRotation.x += this.joystickRotationVecX;
+
+		this.camera.cameraDirection.addInPlace(
+			BABYLON.Vector3.TransformCoordinates(this.joystickMoveVector, BABYLON.Matrix.RotationY(this.camera.rotation.y)),
+		);
+	}
+}