feat: add VSCode to OpenHands runtime and UI (#4745)

Co-authored-by: openhands <openhands@all-hands.dev>
Co-authored-by: Robert Brennan <accounts@rbren.io>

frontend/src/api/open-hands.ts

@@ -8,6 +8,7 @@ import {
   GitHubAccessTokenResponse,
   ErrorResponse,
   GetConfigResponse,
+  GetVSCodeUrlResponse,
 } from "./open-hands.types";
 
 class OpenHands {
@@ -174,6 +175,14 @@ class OpenHands {
       true,
     );
   }
+
+  /**
+   * Get the VSCode URL
+   * @returns VSCode URL
+   */
+  static async getVSCodeUrl(): Promise<GetVSCodeUrlResponse> {
+    return request(`/api/vscode-url`, {}, false, false, 1);
+  }
 }
 
 export default OpenHands;

frontend/src/api/open-hands.types.ts

@@ -46,3 +46,8 @@ export interface GetConfigResponse {
   GITHUB_CLIENT_ID: string;
   POSTHOG_CLIENT_KEY: string;
 }
+
+export interface GetVSCodeUrlResponse {
+  vscode_url: string | null;
+  error?: string;
+}

frontend/src/assets/vscode-alt.svg

@@ -0,0 +1,57 @@
+<svg width="100" height="100" viewBox="0 0 100 100" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0)">
+<g filter="url(#filter0_d)">
+<mask id="mask0" mask-type="alpha" maskUnits="userSpaceOnUse" x="0" y="0" width="100" height="100">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M70.9119 99.5723C72.4869 100.189 74.2828 100.15 75.8725 99.3807L96.4604 89.4231C98.624 88.3771 100 86.1762 100 83.7616V16.2392C100 13.8247 98.624 11.6238 96.4604 10.5774L75.8725 0.619067C73.7862 -0.389991 71.3446 -0.142885 69.5135 1.19527C69.252 1.38636 69.0028 1.59985 68.769 1.83502L29.3551 37.9795L12.1872 24.88C10.5891 23.6607 8.35365 23.7606 6.86938 25.1178L1.36302 30.1525C-0.452603 31.8127 -0.454583 34.6837 1.35854 36.3466L16.2471 50.0001L1.35854 63.6536C-0.454583 65.3164 -0.452603 68.1876 1.36302 69.8477L6.86938 74.8824C8.35365 76.2395 10.5891 76.34 12.1872 75.1201L29.3551 62.0207L68.769 98.1651C69.3925 98.7923 70.1246 99.2645 70.9119 99.5723ZM75.0152 27.1813L45.1092 50.0001L75.0152 72.8189V27.1813Z" fill="white"/>
+</mask>
+<g mask="url(#mask0)">
+<path d="M96.4614 10.593L75.8567 0.62085C73.4717 -0.533437 70.6215 -0.0465506 68.7498 1.83492L1.29834 63.6535C-0.515935 65.3164 -0.513852 68.1875 1.30281 69.8476L6.8125 74.8823C8.29771 76.2395 10.5345 76.339 12.1335 75.1201L93.3604 13.18C96.0854 11.102 100 13.0557 100 16.4939V16.2535C100 13.84 98.6239 11.64 96.4614 10.593Z" fill="#D9D9D9"/>
+<g filter="url(#filter1_d)">
+<path d="M96.4614 89.4074L75.8567 99.3797C73.4717 100.534 70.6215 100.047 68.7498 98.1651L1.29834 36.3464C-0.515935 34.6837 -0.513852 31.8125 1.30281 30.1524L6.8125 25.1177C8.29771 23.7605 10.5345 23.6606 12.1335 24.88L93.3604 86.8201C96.0854 88.8985 100 86.9447 100 83.5061V83.747C100 86.1604 98.6239 88.3603 96.4614 89.4074Z" fill="#E6E6E6"/>
+</g>
+<g filter="url(#filter2_d)">
+<path d="M75.8578 99.3807C73.4721 100.535 70.6219 100.047 68.75 98.1651C71.0564 100.483 75 98.8415 75 95.5631V4.43709C75 1.15852 71.0565 -0.483493 68.75 1.83492C70.6219 -0.0467614 73.4721 -0.534276 75.8578 0.618963L96.4583 10.5773C98.6229 11.6237 100 13.8246 100 16.2391V83.7616C100 86.1762 98.6229 88.3761 96.4583 89.4231L75.8578 99.3807Z" fill="white"/>
+</g>
+<g style="mix-blend-mode:overlay" opacity="0.25">
+<path style="mix-blend-mode:overlay" opacity="0.25" fill-rule="evenodd" clip-rule="evenodd" d="M70.8508 99.5723C72.4258 100.189 74.2218 100.15 75.8115 99.3807L96.4 89.4231C98.5635 88.3771 99.9386 86.1762 99.9386 83.7616V16.2391C99.9386 13.8247 98.5635 11.6239 96.4 10.5774L75.8115 0.618974C73.7252 -0.390085 71.2835 -0.142871 69.4525 1.19518C69.1909 1.38637 68.9418 1.59976 68.7079 1.83493L29.2941 37.9795L12.1261 24.88C10.528 23.6606 8.2926 23.7605 6.80833 25.1177L1.30198 30.1524C-0.51354 31.8126 -0.515625 34.6837 1.2975 36.3465L16.186 50L1.2975 63.6536C-0.515625 65.3164 -0.51354 68.1875 1.30198 69.8476L6.80833 74.8824C8.2926 76.2395 10.528 76.339 12.1261 75.1201L29.2941 62.0207L68.7079 98.1651C69.3315 98.7923 70.0635 99.2645 70.8508 99.5723ZM74.9542 27.1812L45.0481 50L74.9542 72.8188V27.1812Z" fill="url(#paint0_linear)"/>
+</g>
+</g>
+</g>
+</g>
+<defs>
+<filter id="filter0_d" x="-6.25" y="-4.16667" width="112.5" height="112.5" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"/>
+<feOffset dy="2.08333"/>
+<feGaussianBlur stdDeviation="3.125"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.15 0"/>
+<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow" result="shape"/>
+</filter>
+<filter id="filter1_d" x="-8.39436" y="15.6951" width="116.728" height="92.6376" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"/>
+<feOffset/>
+<feGaussianBlur stdDeviation="4.16667"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.25 0"/>
+<feBlend mode="overlay" in2="BackgroundImageFix" result="effect1_dropShadow"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow" result="shape"/>
+</filter>
+<filter id="filter2_d" x="60.4167" y="-8.33346" width="47.9167" height="116.667" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"/>
+<feOffset/>
+<feGaussianBlur stdDeviation="4.16667"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.25 0"/>
+<feBlend mode="overlay" in2="BackgroundImageFix" result="effect1_dropShadow"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow" result="shape"/>
+</filter>
+<linearGradient id="paint0_linear" x1="49.939" y1="-5.19792e-05" x2="49.939" y2="100.001" gradientUnits="userSpaceOnUse">
+<stop stop-color="white"/>
+<stop offset="1" stop-color="white" stop-opacity="0"/>
+</linearGradient>
+<clipPath id="clip0">
+<rect width="100" height="100" fill="white"/>
+</clipPath>
+</defs>
+</svg>

frontend/src/components/file-explorer/FileExplorer.tsx

@@ -12,6 +12,7 @@ import { useTranslation } from "react-i18next";
 import { twMerge } from "tailwind-merge";
 import AgentState from "#/types/AgentState";
 import { setRefreshID } from "#/state/codeSlice";
+import { addAssistantMessage } from "#/state/chatSlice";
 import IconButton from "../IconButton";
 import ExplorerTree from "./ExplorerTree";
 import toast from "#/utils/toast";
@@ -20,6 +21,7 @@ import { I18nKey } from "#/i18n/declaration";
 import OpenHands from "#/api/open-hands";
 import { useFiles } from "#/context/files";
 import { isOpenHandsErrorResponse } from "#/api/open-hands.utils";
+import VSCodeIcon from "#/assets/vscode-alt.svg?react";
 
 interface ExplorerActionsProps {
   onRefresh: () => void;
@@ -168,6 +170,35 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
     }
   };
 
+  const handleVSCodeClick = async (e: React.MouseEvent) => {
+    e.preventDefault();
+    try {
+      const response = await OpenHands.getVSCodeUrl();
+      if (response.vscode_url) {
+        dispatch(
+          addAssistantMessage(
+            "You opened VS Code. Please inform the agent of any changes you made to the workspace or environment. To avoid conflicts, it's best to pause the agent before making any changes.",
+          ),
+        );
+        window.open(response.vscode_url, "_blank");
+      } else {
+        toast.error(
+          `open-vscode-error-${new Date().getTime()}`,
+          t(I18nKey.EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE, {
+            error: response.error,
+          }),
+        );
+      }
+    } catch (exp_error) {
+      toast.error(
+        `open-vscode-error-${new Date().getTime()}`,
+        t(I18nKey.EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE, {
+          error: String(exp_error),
+        }),
+      );
+    }
+  };
+
   React.useEffect(() => {
     refreshWorkspace();
   }, [curAgentState]);
@@ -210,7 +241,7 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
           !isOpen ? "w-12" : "w-60",
         )}
       >
-        <div className="flex flex-col relative h-full px-3 py-2">
+        <div className="flex flex-col relative h-full px-3 py-2 overflow-hidden">
           <div className="sticky top-0 bg-neutral-800">
             <div
               className={twMerge(
@@ -232,7 +263,7 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
             </div>
           </div>
           {!error && (
-            <div className="overflow-auto flex-grow">
+            <div className="overflow-auto flex-grow min-h-0">
               <div style={{ display: !isOpen ? "none" : "block" }}>
                 <ExplorerTree files={paths} />
               </div>
@@ -243,6 +274,27 @@ function FileExplorer({ error, isOpen, onToggle }: FileExplorerProps) {
               <p className="text-neutral-300 text-sm">{error}</p>
             </div>
           )}
+          {isOpen && (
+            <button
+              type="button"
+              onClick={handleVSCodeClick}
+              disabled={
+                curAgentState === AgentState.INIT ||
+                curAgentState === AgentState.LOADING
+              }
+              className={twMerge(
+                "mt-auto mb-2 w-full h-10 text-white rounded flex items-center justify-center gap-2 transition-colors",
+                curAgentState === AgentState.INIT ||
+                  curAgentState === AgentState.LOADING
+                  ? "bg-neutral-600 cursor-not-allowed"
+                  : "bg-[#4465DB] hover:bg-[#3451C7]",
+              )}
+              aria-label="Open in VS Code"
+            >
+              <VSCodeIcon width={20} height={20} />
+              Open in VS Code
+            </button>
+          )}
         </div>
         <input
           data-testid="file-input"

frontend/src/i18n/translation.json

@@ -678,6 +678,16 @@
     "tr": "Sunucudan beklenmeyen yanıt yapısı",
     "no": "Uventet responsstruktur fra serveren"
   },
+  "EXPLORER$VSCODE_SWITCHING_MESSAGE": {
+    "en": "Switching to VS Code in 3 seconds...\nImportant: Please inform the agent of any changes you make in VS Code. To avoid conflicts, wait for the assistant to complete its work before making your own changes.",
+    "zh-CN": "3 秒后切换到 VS Code\n重要提示：请告知 OpenHands 您在 VS Code 中进行的任何更改。为了避免冲突，请在 OpenHands 完成工作后再进行自己的更改。",
+    "zh-TW": "3 秒後切換到 VS Code\n重要提示：請告知 OpenHands 您在 VS Code 中進行的任何更改。為避免衝突，請在 OpenHands 完成工作後再進行自己的更改。"
+  },
+  "EXPLORER$VSCODE_SWITCHING_ERROR_MESSAGE": {
+    "en": "Error switching to VS Code: {{error}}",
+    "zh-CN": "切换到 VS Code 时发生错误: {{error}}",
+    "zh-TW": "切換到 VS Code 時發生錯誤: {{error}}"
+  },
   "LOAD_SESSION$MODAL_TITLE": {
     "en": "Return to existing session?",
     "de": "Zurück zu vorhandener Sitzung?",

frontend/src/utils/toast.tsx

@@ -10,7 +10,6 @@ export default {
       style: {
         background: "#ef4444",
         color: "#fff",
-        lineBreak: "anywhere",
       },
       iconTheme: {
         primary: "#ef4444",
@@ -19,25 +18,20 @@ export default {
     });
     idMap.set(id, toastId);
   },
-  success: (id: string, msg: string) => {
-    const toastId = idMap.get(id);
-    if (toastId === undefined) return;
-    if (toastId) {
-      toast.success(msg, {
-        id: toastId,
-        duration: 4000,
-        style: {
-          background: "#333",
-          color: "#fff",
-          lineBreak: "anywhere",
-        },
-        iconTheme: {
-          primary: "#333",
-          secondary: "#fff",
-        },
-      });
-    }
-    idMap.delete(id);
+  success: (id: string, msg: string, duration: number = 4000) => {
+    if (idMap.has(id)) return; // prevent duplicate toast
+    const toastId = toast.success(msg, {
+      duration,
+      style: {
+        background: "#333",
+        color: "#fff",
+      },
+      iconTheme: {
+        primary: "#333",
+        secondary: "#fff",
+      },
+    });
+    idMap.set(id, toastId);
   },
   settingsChanged: (msg: string) => {
     toast(msg, {
@@ -48,7 +42,6 @@ export default {
       style: {
         background: "#333",
         color: "#fff",
-        lineBreak: "anywhere",
       },
     });
   },

openhands/core/cli.py

@@ -116,6 +116,7 @@ async def main():
         event_stream=event_stream,
         sid=sid,
         plugins=agent_cls.sandbox_plugins,
+        headless_mode=True,
     )
 
     controller = AgentController(

openhands/core/main.py

@@ -54,11 +54,14 @@ def read_task_from_stdin() -> str:
 def create_runtime(
     config: AppConfig,
     sid: str | None = None,
+    headless_mode: bool = True,
 ) -> Runtime:
     """Create a runtime for the agent to run on.
 
     config: The app config.
     sid: The session id.
+    headless_mode: Whether the agent is run in headless mode. `create_runtime` is typically called within evaluation scripts,
+        where we don't want to have the VSCode UI open, so it defaults to True.
     """
     # if sid is provided on the command line, use it as the name of the event stream
     # otherwise generate it on the basis of the configured jwt_secret
@@ -80,6 +83,7 @@ def create_runtime(
         event_stream=event_stream,
         sid=session_id,
         plugins=agent_cls.sandbox_plugins,
+        headless_mode=headless_mode,
     )
 
     return runtime
@@ -122,7 +126,7 @@ async def run_controller(
     sid = sid or generate_sid(config)
 
     if runtime is None:
-        runtime = create_runtime(config, sid=sid)
+        runtime = create_runtime(config, sid=sid, headless_mode=headless_mode)
         await runtime.connect()
 
     event_stream = runtime.event_stream

openhands/runtime/action_execution_server.py

@@ -47,14 +47,11 @@ from openhands.events.observation import (
 from openhands.events.serialization import event_from_dict, event_to_dict
 from openhands.runtime.browser import browse
 from openhands.runtime.browser.browser_env import BrowserEnv
-from openhands.runtime.plugins import (
-    ALL_PLUGINS,
-    JupyterPlugin,
-    Plugin,
-)
+from openhands.runtime.plugins import ALL_PLUGINS, JupyterPlugin, Plugin, VSCodePlugin
 from openhands.runtime.utils.bash import BashSession
 from openhands.runtime.utils.files import insert_lines, read_lines
 from openhands.runtime.utils.runtime_init import init_user_and_working_directory
+from openhands.runtime.utils.system import check_port_available
 from openhands.utils.async_utils import wait_all
 
 
@@ -116,7 +113,10 @@ class ActionExecutor:
         return self._initial_pwd
 
     async def ainit(self):
-        await wait_all(self._init_plugin(plugin) for plugin in self.plugins_to_load)
+        await wait_all(
+            (self._init_plugin(plugin) for plugin in self.plugins_to_load),
+            timeout=30,
+        )
 
         # This is a temporary workaround
         # TODO: refactor AgentSkills to be part of JupyterPlugin
@@ -345,6 +345,8 @@ if __name__ == '__main__':
     )
     # example: python client.py 8000 --working-dir /workspace --plugins JupyterRequirement
     args = parser.parse_args()
+    os.environ['VSCODE_PORT'] = str(int(args.port) + 1)
+    assert check_port_available(int(os.environ['VSCODE_PORT']))
 
     plugins_to_load: list[Plugin] = []
     if args.plugins:
@@ -527,6 +529,19 @@ if __name__ == '__main__':
     async def alive():
         return {'status': 'ok'}
 
+    # ================================
+    # VSCode-specific operations
+    # ================================
+
+    @app.get('/vscode/connection_token')
+    async def get_vscode_connection_token():
+        assert client is not None
+        if 'vscode' in client.plugins:
+            plugin: VSCodePlugin = client.plugins['vscode']  # type: ignore
+            return {'token': plugin.vscode_connection_token}
+        else:
+            return {'token': None}
+
     # ================================
     # File-specific operations for UI
     # ================================

openhands/runtime/base.py

@@ -30,7 +30,11 @@ from openhands.events.observation import (
     UserRejectObservation,
 )
 from openhands.events.serialization.action import ACTION_TYPE_TO_CLASS
-from openhands.runtime.plugins import JupyterRequirement, PluginRequirement
+from openhands.runtime.plugins import (
+    JupyterRequirement,
+    PluginRequirement,
+    VSCodeRequirement,
+)
 from openhands.runtime.utils.edit import FileEditRuntimeMixin
 from openhands.utils.async_utils import call_sync_from_async
 
@@ -84,13 +88,20 @@ class Runtime(FileEditRuntimeMixin):
         env_vars: dict[str, str] | None = None,
         status_callback: Callable | None = None,
         attach_to_existing: bool = False,
+        headless_mode: bool = False,
     ):
         self.sid = sid
         self.event_stream = event_stream
         self.event_stream.subscribe(
             EventStreamSubscriber.RUNTIME, self.on_event, self.sid
         )
-        self.plugins = plugins if plugins is not None and len(plugins) > 0 else []
+        self.plugins = (
+            copy.deepcopy(plugins) if plugins is not None and len(plugins) > 0 else []
+        )
+        # add VSCode plugin if not in headless mode
+        if not headless_mode:
+            self.plugins.append(VSCodeRequirement())
+
         self.status_callback = status_callback
         self.attach_to_existing = attach_to_existing
 
@@ -101,6 +112,10 @@ class Runtime(FileEditRuntimeMixin):
         if env_vars is not None:
             self.initial_env_vars.update(env_vars)
 
+        self._vscode_enabled = any(
+            isinstance(plugin, VSCodeRequirement) for plugin in self.plugins
+        )
+
         # Load mixins
         FileEditRuntimeMixin.__init__(self)
 
@@ -278,3 +293,15 @@ class Runtime(FileEditRuntimeMixin):
     def copy_from(self, path: str) -> Path:
         """Zip all files in the sandbox and return a path in the local filesystem."""
         raise NotImplementedError('This method is not implemented in the base class.')
+
+    # ====================================================================
+    # VSCode
+    # ====================================================================
+
+    @property
+    def vscode_enabled(self) -> bool:
+        return self._vscode_enabled
+
+    @property
+    def vscode_url(self) -> str | None:
+        raise NotImplementedError('This method is not implemented in the base class.')

openhands/runtime/impl/eventstream/eventstream_runtime.py

@@ -136,6 +136,7 @@ class EventStreamRuntime(Runtime):
         env_vars: dict[str, str] | None = None,
         status_callback: Callable | None = None,
         attach_to_existing: bool = False,
+        headless_mode: bool = True,
     ):
         super().__init__(
             config,
@@ -145,6 +146,7 @@ class EventStreamRuntime(Runtime):
             env_vars,
             status_callback,
             attach_to_existing,
+            headless_mode,
         )
 
     def __init__(
@@ -156,10 +158,13 @@ class EventStreamRuntime(Runtime):
         env_vars: dict[str, str] | None = None,
         status_callback: Callable | None = None,
         attach_to_existing: bool = False,
+        headless_mode: bool = True,
     ):
         self.config = config
         self._host_port = 30000  # initial dummy value
         self._container_port = 30001  # initial dummy value
+        self._vscode_url: str | None = None  # initial dummy value
+        self._runtime_initialized: bool = False
         self.api_url = f'{self.config.sandbox.local_runtime_url}:{self._container_port}'
         self.session = requests.Session()
         self.status_callback = status_callback
@@ -190,6 +195,7 @@ class EventStreamRuntime(Runtime):
             env_vars,
             status_callback,
             attach_to_existing,
+            headless_mode,
         )
 
     async def connect(self):
@@ -221,7 +227,10 @@ class EventStreamRuntime(Runtime):
                 'info', f'Starting runtime with image: {self.runtime_container_image}'
             )
             await call_sync_from_async(self._init_container)
-            self.log('info', f'Container started: {self.container_name}')
+            self.log(
+                'info',
+                f'Container started: {self.container_name}. VSCode URL: {self.vscode_url}',
+            )
 
         if not self.attach_to_existing:
             self.log('info', f'Waiting for client to become ready at {self.api_url}...')
@@ -237,10 +246,11 @@ class EventStreamRuntime(Runtime):
 
         self.log(
             'debug',
-            f'Container initialized with plugins: {[plugin.name for plugin in self.plugins]}',
+            f'Container initialized with plugins: {[plugin.name for plugin in self.plugins]}. VSCode URL: {self.vscode_url}',
         )
         if not self.attach_to_existing:
             self.send_status_message(' ')
+        self._runtime_initialized = True
 
     @staticmethod
     @lru_cache(maxsize=1)
@@ -261,7 +271,6 @@ class EventStreamRuntime(Runtime):
             plugin_arg = (
                 f'--plugins {" ".join([plugin.name for plugin in self.plugins])} '
             )
-
         self._host_port = self._find_available_port()
         self._container_port = (
             self._host_port
@@ -270,6 +279,7 @@ class EventStreamRuntime(Runtime):
 
         use_host_network = self.config.sandbox.use_host_network
         network_mode: str | None = 'host' if use_host_network else None
+
         port_mapping: dict[str, list[dict[str, str]]] | None = (
             None
             if use_host_network
@@ -290,6 +300,13 @@ class EventStreamRuntime(Runtime):
         if self.config.debug or DEBUG:
             environment['DEBUG'] = 'true'
 
+        if self.vscode_enabled:
+            # vscode is on port +1 from container port
+            if isinstance(port_mapping, dict):
+                port_mapping[f'{self._container_port + 1}/tcp'] = [
+                    {'HostPort': str(self._host_port + 1)}
+                ]
+
         self.log('debug', f'Workspace Base: {self.config.workspace_base}')
         if (
             self.config.workspace_mount_path is not None
@@ -630,3 +647,30 @@ class EventStreamRuntime(Runtime):
                 return port
         # If no port is found after max_attempts, return the last tried port
         return port
+
+    @property
+    def vscode_url(self) -> str | None:
+        if self.vscode_enabled and self._runtime_initialized:
+            if (
+                hasattr(self, '_vscode_url') and self._vscode_url is not None
+            ):  # cached value
+                return self._vscode_url
+
+            response = send_request(
+                self.session,
+                'GET',
+                f'{self.api_url}/vscode/connection_token',
+                timeout=10,
+            )
+            response_json = response.json()
+            assert isinstance(response_json, dict)
+            if response_json['token'] is None:
+                return None
+            self._vscode_url = f'http://localhost:{self._host_port + 1}/?tkn={response_json["token"]}&folder={self.config.workspace_mount_path_in_sandbox}'
+            self.log(
+                'debug',
+                f'VSCode URL: {self._vscode_url}',
+            )
+            return self._vscode_url
+        else:
+            return None

openhands/runtime/impl/modal/modal_runtime.py

@@ -77,6 +77,7 @@ class ModalRuntime(EventStreamRuntime):
         env_vars: dict[str, str] | None = None,
         status_callback: Callable | None = None,
         attach_to_existing: bool = False,
+        headless_mode: bool = True,
     ):
         assert config.modal_api_token_id, 'Modal API token id is required'
         assert config.modal_api_token_secret, 'Modal API token secret is required'
@@ -124,6 +125,7 @@ class ModalRuntime(EventStreamRuntime):
             env_vars,
             status_callback,
             attach_to_existing,
+            headless_mode,
         )
 
     async def connect(self):

openhands/runtime/impl/remote/remote_runtime.py

@@ -3,6 +3,7 @@ import tempfile
 import threading
 from pathlib import Path
 from typing import Callable, Optional
+from urllib.parse import urlparse
 from zipfile import ZipFile
 
 import requests
@@ -57,6 +58,7 @@ class RemoteRuntime(Runtime):
         env_vars: dict[str, str] | None = None,
         status_callback: Optional[Callable] = None,
         attach_to_existing: bool = False,
+        headless_mode: bool = True,
     ):
         # We need to set session and action_semaphore before the __init__ below, or we get odd errors
         self.session = requests.Session()
@@ -70,6 +72,7 @@ class RemoteRuntime(Runtime):
             env_vars,
             status_callback,
             attach_to_existing,
+            headless_mode,
         )
         if self.config.sandbox.api_key is None:
             raise ValueError(
@@ -89,6 +92,8 @@ class RemoteRuntime(Runtime):
         )
         self.runtime_id: str | None = None
         self.runtime_url: str | None = None
+        self._runtime_initialized: bool = False
+        self._vscode_url: str | None = None  # initial dummy value
 
     async def connect(self):
         try:
@@ -97,6 +102,7 @@ class RemoteRuntime(Runtime):
             self.log('error', 'Runtime failed to start, timed out before ready')
             raise
         await call_sync_from_async(self.setup_initial_env)
+        self._runtime_initialized = True
 
     def _start_or_attach_to_runtime(self):
         existing_runtime = self._check_existing_runtime()
@@ -265,6 +271,43 @@ class RemoteRuntime(Runtime):
                 {'X-Session-API-Key': start_response['session_api_key']}
             )
 
+    @property
+    def vscode_url(self) -> str | None:
+        if self.vscode_enabled and self._runtime_initialized:
+            if (
+                hasattr(self, '_vscode_url') and self._vscode_url is not None
+            ):  # cached value
+                return self._vscode_url
+
+            response = self._send_request(
+                'GET',
+                f'{self.runtime_url}/vscode/connection_token',
+                timeout=10,
+            )
+            response_json = response.json()
+            assert isinstance(response_json, dict)
+            if response_json['token'] is None:
+                return None
+            # parse runtime_url to get vscode_url
+            _parsed_url = urlparse(self.runtime_url)
+            assert isinstance(_parsed_url.scheme, str) and isinstance(
+                _parsed_url.netloc, str
+            )
+            self._vscode_url = f'{_parsed_url.scheme}://vscode-{_parsed_url.netloc}/?tkn={response_json["token"]}&folder={self.config.workspace_mount_path_in_sandbox}'
+            self.log(
+                'debug',
+                f'VSCode URL: {self._vscode_url}',
+            )
+            return self._vscode_url
+        else:
+            return None
+
+    @tenacity.retry(
+        stop=tenacity.stop_after_delay(180) | stop_if_should_exit(),
+        reraise=True,
+        retry=tenacity.retry_if_exception_type(RuntimeNotReadyError),
+        wait=tenacity.wait_fixed(2),
+    )
     def _wait_until_alive(self):
         retry_decorator = tenacity.retry(
             stop=tenacity.stop_after_delay(

openhands/runtime/plugins/__init__.py

@@ -5,6 +5,7 @@ from openhands.runtime.plugins.agent_skills import (
 )
 from openhands.runtime.plugins.jupyter import JupyterPlugin, JupyterRequirement
 from openhands.runtime.plugins.requirement import Plugin, PluginRequirement
+from openhands.runtime.plugins.vscode import VSCodePlugin, VSCodeRequirement
 
 __all__ = [
     'Plugin',
@@ -13,9 +14,12 @@ __all__ = [
     'AgentSkillsPlugin',
     'JupyterRequirement',
     'JupyterPlugin',
+    'VSCodeRequirement',
+    'VSCodePlugin',
 ]
 
 ALL_PLUGINS = {
     'jupyter': JupyterPlugin,
     'agent_skills': AgentSkillsPlugin,
+    'vscode': VSCodePlugin,
 }

openhands/runtime/plugins/vscode/__init__.py

@@ -0,0 +1,51 @@
+import os
+import subprocess
+import time
+import uuid
+from dataclasses import dataclass
+
+from openhands.core.logger import openhands_logger as logger
+from openhands.runtime.plugins.requirement import Plugin, PluginRequirement
+from openhands.runtime.utils.shutdown_listener import should_continue
+from openhands.runtime.utils.system import check_port_available
+
+
+@dataclass
+class VSCodeRequirement(PluginRequirement):
+    name: str = 'vscode'
+
+
+class VSCodePlugin(Plugin):
+    name: str = 'vscode'
+
+    async def initialize(self, username: str):
+        self.vscode_port = int(os.environ['VSCODE_PORT'])
+        self.vscode_connection_token = str(uuid.uuid4())
+        assert check_port_available(self.vscode_port)
+        cmd = (
+            f"su - {username} -s /bin/bash << 'EOF'\n"
+            f'sudo chown -R {username}:{username} /openhands/.openvscode-server\n'
+            'cd /workspace\n'
+            f'exec /openhands/.openvscode-server/bin/openvscode-server --host 0.0.0.0 --connection-token {self.vscode_connection_token} --port {self.vscode_port}\n'
+            'EOF'
+        )
+        print(cmd)
+        self.gateway_process = subprocess.Popen(
+            cmd,
+            stderr=subprocess.STDOUT,
+            shell=True,
+        )
+        # read stdout until the kernel gateway is ready
+        output = ''
+        while should_continue() and self.gateway_process.stdout is not None:
+            line = self.gateway_process.stdout.readline().decode('utf-8')
+            print(line)
+            output += line
+            if 'at' in line:
+                break
+            time.sleep(1)
+            logger.debug('Waiting for VSCode server to start...')
+
+        logger.debug(
+            f'VSCode server started at port {self.vscode_port}. Output: {output}'
+        )

openhands/runtime/utils/runtime_templates/Dockerfile.j2

@@ -1,8 +1,71 @@
 FROM {{ base_image }}
 
-# Shared environment variables (regardless of init or not)
-ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry
-ENV MAMBA_ROOT_PREFIX=/openhands/micromamba
+# Shared environment variables
+ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry \
+    MAMBA_ROOT_PREFIX=/openhands/micromamba \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    EDITOR=code \
+    VISUAL=code \
+    GIT_EDITOR="code --wait" \
+    OPENVSCODE_SERVER_ROOT=/openhands/.openvscode-server
+
+{% macro setup_base_system() %}
+
+# Install base system dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        wget curl sudo apt-utils git \
+        {% if 'ubuntu' in base_image and (base_image.endswith(':latest') or base_image.endswith(':24.04')) %}
+        libgl1 \
+        {% else %}
+        libgl1-mesa-glx \
+        {% endif %}
+        libasound2-plugins libatomic1 curl && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Remove UID 1000 if it's called pn--this fixes the nikolaik image for ubuntu users
+RUN if getent passwd 1000 | grep -q pn; then userdel pn; fi
+
+# Create necessary directories
+RUN mkdir -p /openhands && \
+    mkdir -p /openhands/logs && \
+    mkdir -p /openhands/poetry
+
+{% endmacro %}
+
+{% macro setup_vscode_server() %}
+# Reference:
+# 1. https://github.com/gitpod-io/openvscode-server
+# 2. https://github.com/gitpod-io/openvscode-releases
+
+# Setup VSCode Server
+ARG RELEASE_TAG="openvscode-server-v1.94.2"
+ARG RELEASE_ORG="gitpod-io"
+# ARG USERNAME=openvscode-server
+# ARG USER_UID=1000
+# ARG USER_GID=1000
+
+RUN if [ -z "${RELEASE_TAG}" ]; then \
+        echo "The RELEASE_TAG build arg must be set." >&2 && \
+        exit 1; \
+    fi && \
+    arch=$(uname -m) && \
+    if [ "${arch}" = "x86_64" ]; then \
+        arch="x64"; \
+    elif [ "${arch}" = "aarch64" ]; then \
+        arch="arm64"; \
+    elif [ "${arch}" = "armv7l" ]; then \
+        arch="armhf"; \
+    fi && \
+    wget https://github.com/${RELEASE_ORG}/openvscode-server/releases/download/${RELEASE_TAG}/${RELEASE_TAG}-linux-${arch}.tar.gz && \
+    tar -xzf ${RELEASE_TAG}-linux-${arch}.tar.gz && \
+    mv -f ${RELEASE_TAG}-linux-${arch} ${OPENVSCODE_SERVER_ROOT} && \
+    cp ${OPENVSCODE_SERVER_ROOT}/bin/remote-cli/openvscode-server ${OPENVSCODE_SERVER_ROOT}/bin/remote-cli/code && \
+    rm -f ${RELEASE_TAG}-linux-${arch}.tar.gz
+
+{% endmacro %}
 
 {% macro install_dependencies() %}
 # Install all dependencies
@@ -28,6 +91,7 @@ RUN \
     # Clean up
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     /openhands/micromamba/bin/micromamba clean --all
+
 {% endmacro %}
 
 {% if build_from_scratch %}
@@ -37,25 +101,8 @@ RUN \
 # This is used in cases where the base image is something more generic like nikolaik/python-nodejs
 # rather than the current OpenHands release
 
-{% if 'ubuntu' in base_image and (base_image.endswith(':latest') or base_image.endswith(':24.04')) %}
-{% set LIBGL_MESA = 'libgl1' %}
-{% else %}
-{% set LIBGL_MESA = 'libgl1-mesa-glx' %}
-{% endif %}
-
-# Install necessary packages and clean up in one layer
-RUN apt-get update && \
-    apt-get install -y wget curl sudo apt-utils {{ LIBGL_MESA }} libasound2-plugins git && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Remove UID 1000 if it's called pn--this fixes the nikolaik image for ubuntu users
-RUN if getent passwd 1000 | grep -q pn; then userdel pn; fi
-
-# Create necessary directories
-RUN mkdir -p /openhands && \
-    mkdir -p /openhands/logs && \
-    mkdir -p /openhands/poetry
+{{ setup_base_system() }}
+{{ setup_vscode_server() }}
 
 # Install micromamba
 RUN mkdir -p /openhands/micromamba/bin && \
@@ -72,6 +119,7 @@ RUN \
     if [ -d /openhands/code ]; then rm -rf /openhands/code; fi && \
     mkdir -p /openhands/code/openhands && \
     touch /openhands/code/openhands/__init__.py
+
 COPY ./code/pyproject.toml ./code/poetry.lock /openhands/code/
 
 {{ install_dependencies() }}

openhands/runtime/utils/system.py

@@ -3,6 +3,18 @@ import socket
 import time
 
 
+def check_port_available(port: int) -> bool:
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        sock.bind(('localhost', port))
+        return True
+    except OSError:
+        time.sleep(0.1)  # Short delay to further reduce chance of collisions
+        return False
+    finally:
+        sock.close()
+
+
 def find_available_tcp_port(min_port=30000, max_port=39999, max_attempts=10) -> int:
     """Find an available TCP port in a specified range.
 
@@ -19,15 +31,8 @@ def find_available_tcp_port(min_port=30000, max_port=39999, max_attempts=10) ->
     rng.shuffle(ports)
 
     for port in ports[:max_attempts]:
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        try:
-            sock.bind(('localhost', port))
+        if check_port_available(port):
             return port
-        except OSError:
-            time.sleep(0.1)  # Short delay to further reduce chance of collisions
-            continue
-        finally:
-            sock.close()
     return -1
 
 

openhands/server/listen.py

@@ -892,6 +892,34 @@ async def authenticate(request: Request):
     return response
 
 
+@app.get('/api/vscode-url')
+async def get_vscode_url(request: Request):
+    """Get the VSCode URL.
+
+    This endpoint allows getting the VSCode URL.
+
+    Args:
+        request (Request): The incoming FastAPI request object.
+
+    Returns:
+        JSONResponse: A JSON response indicating the success of the operation.
+    """
+    try:
+        runtime: Runtime = request.state.conversation.runtime
+        logger.debug(f'Runtime type: {type(runtime)}')
+        logger.debug(f'Runtime VSCode URL: {runtime.vscode_url}')
+        return JSONResponse(status_code=200, content={'vscode_url': runtime.vscode_url})
+    except Exception as e:
+        logger.error(f'Error getting VSCode URL: {e}', exc_info=True)
+        return JSONResponse(
+            status_code=500,
+            content={
+                'vscode_url': None,
+                'error': f'Error getting VSCode URL: {e}',
+            },
+        )
+
+
 class SPAStaticFiles(StaticFiles):
     async def get_response(self, path: str, scope):
         try:

openhands/server/session/agent_session.py

@@ -189,6 +189,7 @@ class AgentSession:
             sid=self.sid,
             plugins=agent.sandbox_plugins,
             status_callback=self._status_callback,
+            headless_mode=False,
         )
 
         try:

openhands/server/session/conversation.py

@@ -36,6 +36,7 @@ class Conversation:
             event_stream=self.event_stream,
             sid=self.sid,
             attach_to_existing=True,
+            headless_mode=False,
         )
 
     async def connect(self):

tests/runtime/test_stress_remote_runtime.py

@@ -137,7 +137,7 @@ def process_instance(
     else:
         logger.info(f'Starting evaluation for instance {instance.instance_id}.')
 
-    runtime = create_runtime(config)
+    runtime = create_runtime(config, headless_mode=False)
     call_async_from_sync(runtime.connect)
 
     try:

tests/unit/test_runtime_build.py

@@ -135,7 +135,7 @@ def test_generate_dockerfile_build_from_scratch():
     )
     assert base_image in dockerfile_content
     assert 'apt-get update' in dockerfile_content
-    assert 'apt-get install -y wget curl sudo apt-utils' in dockerfile_content
+    assert 'wget curl sudo apt-utils git' in dockerfile_content
     assert 'poetry' in dockerfile_content and '-c conda-forge' in dockerfile_content
     assert 'python=3.12' in dockerfile_content
 
@@ -155,7 +155,7 @@ def test_generate_dockerfile_build_from_lock():
     )
 
     # These commands SHOULD NOT include in the dockerfile if build_from_scratch is False
-    assert 'RUN apt update && apt install -y wget sudo' not in dockerfile_content
+    assert 'wget curl sudo apt-utils git' not in dockerfile_content
     assert '-c conda-forge' not in dockerfile_content
     assert 'python=3.12' not in dockerfile_content
     assert 'https://micro.mamba.pm/install.sh' not in dockerfile_content
@@ -173,7 +173,7 @@ def test_generate_dockerfile_build_from_versioned():
     )
 
     # these commands should not exist when build from versioned
-    assert 'RUN apt update && apt install -y wget sudo' not in dockerfile_content
+    assert 'wget curl sudo apt-utils git' not in dockerfile_content
     assert '-c conda-forge' not in dockerfile_content
     assert 'python=3.12' not in dockerfile_content
     assert 'https://micro.mamba.pm/install.sh' not in dockerfile_content