debugging keyboard event issues

websocket_pipeline.py

@@ -30,47 +30,72 @@ class WebSocketStreamingPipeline(CausalInferenceStreamingPipeline):
         Get current action from stored WebSocket data instead of stdin
         Returns the same format as get_current_action()
         
-        WebSocket keyboard format: [forward, back, left, right, jump, attack]
-        Pipeline expects different formats per mode:
-        - universal: keyboard [forward, back, left, right], mouse [x, y]
-        - gta_drive: keyboard [forward, back], mouse [x, y] 
-        - templerun: keyboard [left, right], no mouse
+        The original pipeline expects SINGLE ACTION vectors, not multi-action states:
+        - Universal: keyboard [1,0,0,0] = forward only, [0,0,1,0] = left only
+        - WebSocket gives: [forward, back, left, right, jump, attack] with multiple 1s
+        
+        We need to convert multi-action to dominant single action.
         """
-        # Convert WebSocket keyboard format to mode-specific format
+        # Get WebSocket format: [forward, back, left, right, jump, attack]
         if self.current_keyboard is None:
-            ws_keyboard = [0, 0, 0, 0, 0, 0]  # Default WebSocket format
+            ws_keyboard = [0, 0, 0, 0, 0, 0]
         else:
             ws_keyboard = self.current_keyboard
             
-        # Map to mode-specific format
+        if self.current_mouse is None:
+            ws_mouse = [0, 0]
+        else:
+            ws_mouse = self.current_mouse
+        
+        # Convert WebSocket multi-action to single dominant action
         if mode == 'universal':
-            # Use first 4 values: forward, back, left, right
-            keyboard = ws_keyboard[:4]
+            # Pipeline expects: [forward, back, left, right] as single action
+            # Priority order: forward > back > left > right > no action
+            if ws_keyboard[0]:  # forward
+                keyboard = [1, 0, 0, 0]
+            elif ws_keyboard[1]:  # back  
+                keyboard = [0, 1, 0, 0]
+            elif ws_keyboard[2]:  # left
+                keyboard = [0, 0, 1, 0]
+            elif ws_keyboard[3]:  # right
+                keyboard = [0, 0, 0, 1]
+            else:  # no action
+                keyboard = [0, 0, 0, 0]
+                
         elif mode == 'gta_drive':
-            # Use forward and back only
-            keyboard = [ws_keyboard[0], ws_keyboard[1]]
+            # Pipeline expects: [forward, back] as single action
+            if ws_keyboard[0]:  # forward
+                keyboard = [1, 0]
+            elif ws_keyboard[1]:  # back
+                keyboard = [0, 1]
+            else:  # no action
+                keyboard = [0, 0]
+                
         elif mode == 'templerun':
-            # Use left and right only
-            keyboard = [ws_keyboard[2], ws_keyboard[3]]
+            # Pipeline expects: [left, right] as single action
+            if ws_keyboard[2]:  # left
+                keyboard = [1, 0]
+            elif ws_keyboard[3]:  # right
+                keyboard = [0, 1]
+            else:  # no action
+                keyboard = [0, 0]
         else:
-            # Default to universal format
-            keyboard = ws_keyboard[:4]
+            # Default to universal
+            keyboard = [0, 0, 0, 0]
             
-        # Handle mouse (not used in templerun)
+        # Mouse handling - use raw WebSocket values for most modes
         if mode == 'templerun':
-            # Temple Run doesn't use mouse, but return zeros for compatibility
+            # Temple Run doesn't use mouse
             mouse = [0, 0]
         else:
-            if self.current_mouse is None:
-                mouse = [0, 0]
-            else:
-                mouse = self.current_mouse
+            # Use WebSocket mouse values directly (they should be in [-1, 1] range)
+            mouse = ws_mouse
             
-        # Convert to tensors in the format expected by the pipeline
+        # Convert to tensors
         mouse_tensor = torch.tensor(mouse, dtype=torch.float32).cuda()
         keyboard_tensor = torch.tensor(keyboard, dtype=torch.float32).cuda()
         
-        logger.debug(f"WebSocket action for mode {mode}: keyboard={keyboard}, mouse={mouse}")
+        logger.debug(f"WebSocket action for mode {mode}: ws_kb={ws_keyboard}, ws_mouse={ws_mouse} -> kb={keyboard}, mouse={mouse}")
         
         return {
             'mouse': mouse_tensor,