|
|
@@ -1,410 +0,0 @@
|
|
|
-import atexit
|
|
|
-import os
|
|
|
-import shlex
|
|
|
-import sys
|
|
|
-import tarfile
|
|
|
-import time
|
|
|
-import uuid
|
|
|
-from collections import namedtuple
|
|
|
-from glob import glob
|
|
|
-
|
|
|
-import docker
|
|
|
-
|
|
|
-from opendevin.core.config import config
|
|
|
-from opendevin.core.const.guide_url import TROUBLESHOOTING_URL
|
|
|
-from opendevin.core.exceptions import SandboxInvalidBackgroundCommandError
|
|
|
-from opendevin.core.logger import opendevin_logger as logger
|
|
|
-from opendevin.core.schema import CancellableStream
|
|
|
-from opendevin.runtime.docker.process import DockerProcess, Process
|
|
|
-from opendevin.runtime.sandbox import Sandbox
|
|
|
-
|
|
|
-ExecResult = namedtuple('ExecResult', 'exit_code,output')
|
|
|
-""" A result of Container.exec_run with the properties ``exit_code`` and
|
|
|
- ``output``. """
|
|
|
-
|
|
|
-
|
|
|
-class DockerExecCancellableStream(CancellableStream):
|
|
|
- # Reference: https://github.com/docker/docker-py/issues/1989
|
|
|
- def __init__(self, _client, _id, _output):
|
|
|
- super().__init__(self.read_output())
|
|
|
- self._id = _id
|
|
|
- self._client = _client
|
|
|
- self._output = _output
|
|
|
-
|
|
|
- def close(self):
|
|
|
- self.closed = True
|
|
|
-
|
|
|
- def exit_code(self):
|
|
|
- return self.inspect()['ExitCode']
|
|
|
-
|
|
|
- def inspect(self):
|
|
|
- return self._client.api.exec_inspect(self._id)
|
|
|
-
|
|
|
- def read_output(self):
|
|
|
- for chunk in self._output:
|
|
|
- yield chunk.decode('utf-8')
|
|
|
-
|
|
|
-
|
|
|
-def container_exec_run(
|
|
|
- container,
|
|
|
- cmd,
|
|
|
- stdout=True,
|
|
|
- stderr=True,
|
|
|
- stdin=False,
|
|
|
- tty=False,
|
|
|
- privileged=False,
|
|
|
- user='',
|
|
|
- detach=False,
|
|
|
- stream=False,
|
|
|
- socket=False,
|
|
|
- environment=None,
|
|
|
- workdir=None,
|
|
|
-) -> ExecResult:
|
|
|
- exec_id = container.client.api.exec_create(
|
|
|
- container.id,
|
|
|
- cmd,
|
|
|
- stdout=stdout,
|
|
|
- stderr=stderr,
|
|
|
- stdin=stdin,
|
|
|
- tty=tty,
|
|
|
- privileged=privileged,
|
|
|
- user=user,
|
|
|
- environment=environment,
|
|
|
- workdir=workdir,
|
|
|
- )['Id']
|
|
|
-
|
|
|
- output = container.client.api.exec_start(
|
|
|
- exec_id, detach=detach, tty=tty, stream=stream, socket=socket
|
|
|
- )
|
|
|
-
|
|
|
- if stream:
|
|
|
- return ExecResult(
|
|
|
- None, DockerExecCancellableStream(container.client, exec_id, output)
|
|
|
- )
|
|
|
-
|
|
|
- if socket:
|
|
|
- return ExecResult(None, output)
|
|
|
-
|
|
|
- return ExecResult(container.client.api.exec_inspect(exec_id)['ExitCode'], output)
|
|
|
-
|
|
|
-
|
|
|
-class DockerExecBox(Sandbox):
|
|
|
- instance_id: str
|
|
|
- container_image: str
|
|
|
- container_name_prefix = 'opendevin-sandbox-'
|
|
|
- container_name: str
|
|
|
- container: docker.models.containers.Container
|
|
|
- docker_client: docker.DockerClient
|
|
|
-
|
|
|
- cur_background_id = 0
|
|
|
- background_commands: dict[int, Process] = {}
|
|
|
-
|
|
|
- def __init__(
|
|
|
- self,
|
|
|
- container_image: str | None = None,
|
|
|
- timeout: int = config.sandbox_timeout,
|
|
|
- sid: str | None = None,
|
|
|
- ):
|
|
|
- # Initialize docker client. Throws an exception if Docker is not reachable.
|
|
|
- try:
|
|
|
- self.docker_client = docker.from_env()
|
|
|
- except Exception as ex:
|
|
|
- logger.exception(
|
|
|
- f'Error creating controller. Please check Docker is running and visit `{TROUBLESHOOTING_URL}` for more debugging information.',
|
|
|
- exc_info=False,
|
|
|
- )
|
|
|
- raise ex
|
|
|
-
|
|
|
- self.instance_id = (
|
|
|
- sid + str(uuid.uuid4()) if sid is not None else str(uuid.uuid4())
|
|
|
- )
|
|
|
-
|
|
|
- # TODO: this timeout is actually essential - need a better way to set it
|
|
|
- # if it is too short, the container may still waiting for previous
|
|
|
- # command to finish (e.g. apt-get update)
|
|
|
- # if it is too long, the user may have to wait for a unnecessary long time
|
|
|
- self.timeout = timeout
|
|
|
- self.container_image = (
|
|
|
- config.sandbox_container_image
|
|
|
- if container_image is None
|
|
|
- else container_image
|
|
|
- )
|
|
|
- self.container_name = self.container_name_prefix + self.instance_id
|
|
|
-
|
|
|
- logger.info(
|
|
|
- 'Starting Docker container with image %s, sandbox workspace dir=%s',
|
|
|
- self.container_image,
|
|
|
- self.sandbox_workspace_dir,
|
|
|
- )
|
|
|
-
|
|
|
- # always restart the container, cuz the initial be regarded as a new session
|
|
|
- self.restart_docker_container()
|
|
|
-
|
|
|
- if self.run_as_devin:
|
|
|
- self.setup_devin_user()
|
|
|
- atexit.register(self.close)
|
|
|
- super().__init__()
|
|
|
-
|
|
|
- def setup_devin_user(self):
|
|
|
- cmds = [
|
|
|
- f'useradd --shell /bin/bash -u {self.user_id} -o -c "" -m devin',
|
|
|
- r"echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers",
|
|
|
- 'sudo adduser devin sudo',
|
|
|
- ]
|
|
|
- for cmd in cmds:
|
|
|
- exit_code, logs = self.container.exec_run(
|
|
|
- ['/bin/bash', '-c', cmd],
|
|
|
- workdir=self.sandbox_workspace_dir,
|
|
|
- environment=self._env,
|
|
|
- )
|
|
|
- if exit_code != 0:
|
|
|
- raise Exception(f'Failed to setup devin user: {logs}')
|
|
|
-
|
|
|
- def get_exec_cmd(self, cmd: str) -> list[str]:
|
|
|
- if self.run_as_devin:
|
|
|
- return ['su', 'devin', '-c', cmd]
|
|
|
- else:
|
|
|
- return ['/bin/bash', '-c', cmd]
|
|
|
-
|
|
|
- def read_logs(self, id) -> str:
|
|
|
- if id not in self.background_commands:
|
|
|
- raise SandboxInvalidBackgroundCommandError()
|
|
|
- bg_cmd = self.background_commands[id]
|
|
|
- return bg_cmd.read_logs()
|
|
|
-
|
|
|
- def execute(
|
|
|
- self, cmd: str, stream: bool = False, timeout: int | None = None
|
|
|
- ) -> tuple[int, str | CancellableStream]:
|
|
|
- timeout = timeout if timeout is not None else self.timeout
|
|
|
- wrapper = f'timeout {self.timeout}s bash -c {shlex.quote(cmd)}'
|
|
|
- _exit_code, _output = container_exec_run(
|
|
|
- self.container,
|
|
|
- wrapper,
|
|
|
- stream=stream,
|
|
|
- workdir=self.sandbox_workspace_dir,
|
|
|
- environment=self._env,
|
|
|
- )
|
|
|
-
|
|
|
- if stream:
|
|
|
- return _exit_code, _output
|
|
|
-
|
|
|
- print(_output)
|
|
|
- _output = _output.decode('utf-8')
|
|
|
- if _output.endswith('\n'):
|
|
|
- _output = _output[:-1]
|
|
|
- return _exit_code, _output
|
|
|
-
|
|
|
- def copy_to(self, host_src: str, sandbox_dest: str, recursive: bool = False):
|
|
|
- # mkdir -p sandbox_dest if it doesn't exist
|
|
|
- exit_code, logs = self.container.exec_run(
|
|
|
- ['/bin/bash', '-c', f'mkdir -p {sandbox_dest}'],
|
|
|
- workdir=self.sandbox_workspace_dir,
|
|
|
- environment=self._env,
|
|
|
- )
|
|
|
- if exit_code != 0:
|
|
|
- raise Exception(
|
|
|
- f'Failed to create directory {sandbox_dest} in sandbox: {logs}'
|
|
|
- )
|
|
|
-
|
|
|
- if recursive:
|
|
|
- assert os.path.isdir(
|
|
|
- host_src
|
|
|
- ), 'Source must be a directory when recursive is True'
|
|
|
- files = glob(host_src + '/**/*', recursive=True)
|
|
|
- srcname = os.path.basename(host_src)
|
|
|
- tar_filename = os.path.join(os.path.dirname(host_src), srcname + '.tar')
|
|
|
- with tarfile.open(tar_filename, mode='w') as tar:
|
|
|
- for file in files:
|
|
|
- tar.add(
|
|
|
- file, arcname=os.path.relpath(file, os.path.dirname(host_src))
|
|
|
- )
|
|
|
- else:
|
|
|
- assert os.path.isfile(
|
|
|
- host_src
|
|
|
- ), 'Source must be a file when recursive is False'
|
|
|
- srcname = os.path.basename(host_src)
|
|
|
- tar_filename = os.path.join(os.path.dirname(host_src), srcname + '.tar')
|
|
|
- with tarfile.open(tar_filename, mode='w') as tar:
|
|
|
- tar.add(host_src, arcname=srcname)
|
|
|
-
|
|
|
- with open(tar_filename, 'rb') as f:
|
|
|
- data = f.read()
|
|
|
-
|
|
|
- self.container.put_archive(os.path.dirname(sandbox_dest), data)
|
|
|
- os.remove(tar_filename)
|
|
|
-
|
|
|
- def execute_in_background(self, cmd: str) -> Process:
|
|
|
- result = self.container.exec_run(
|
|
|
- self.get_exec_cmd(cmd),
|
|
|
- socket=True,
|
|
|
- workdir=self.sandbox_workspace_dir,
|
|
|
- environment=self._env,
|
|
|
- )
|
|
|
- result.output._sock.setblocking(0)
|
|
|
- pid = self.get_pid(cmd)
|
|
|
- bg_cmd = DockerProcess(self.cur_background_id, cmd, result, pid)
|
|
|
- self.background_commands[bg_cmd.pid] = bg_cmd
|
|
|
- self.cur_background_id += 1
|
|
|
- return bg_cmd
|
|
|
-
|
|
|
- def get_pid(self, cmd):
|
|
|
- exec_result = self.container.exec_run('ps aux', environment=self._env)
|
|
|
- processes = exec_result.output.decode('utf-8').splitlines()
|
|
|
- cmd = ' '.join(self.get_exec_cmd(cmd))
|
|
|
-
|
|
|
- for process in processes:
|
|
|
- if cmd in process:
|
|
|
- pid = process.split()[1] # second column is the pid
|
|
|
- return pid
|
|
|
- return None
|
|
|
-
|
|
|
- def kill_background(self, id: int) -> Process:
|
|
|
- if id not in self.background_commands:
|
|
|
- raise SandboxInvalidBackgroundCommandError()
|
|
|
- bg_cmd = self.background_commands[id]
|
|
|
- if bg_cmd.pid is not None:
|
|
|
- self.container.exec_run(
|
|
|
- f'kill -9 {bg_cmd.pid}',
|
|
|
- workdir=self.sandbox_workspace_dir,
|
|
|
- environment=self._env,
|
|
|
- )
|
|
|
- assert isinstance(bg_cmd, DockerProcess)
|
|
|
- bg_cmd.result.output.close()
|
|
|
- self.background_commands.pop(id)
|
|
|
- return bg_cmd
|
|
|
-
|
|
|
- def stop_docker_container(self):
|
|
|
- try:
|
|
|
- container = self.docker_client.containers.get(self.container_name)
|
|
|
- container.stop()
|
|
|
- container.remove()
|
|
|
- elapsed = 0
|
|
|
- while container.status != 'exited':
|
|
|
- time.sleep(1)
|
|
|
- elapsed += 1
|
|
|
- if elapsed > self.timeout:
|
|
|
- break
|
|
|
- container = self.docker_client.containers.get(self.container_name)
|
|
|
- except docker.errors.NotFound:
|
|
|
- pass
|
|
|
-
|
|
|
- def is_container_running(self):
|
|
|
- try:
|
|
|
- container = self.docker_client.containers.get(self.container_name)
|
|
|
- if container.status == 'running':
|
|
|
- self.container = container
|
|
|
- return True
|
|
|
- return False
|
|
|
- except docker.errors.NotFound:
|
|
|
- return False
|
|
|
-
|
|
|
- def restart_docker_container(self):
|
|
|
- try:
|
|
|
- self.stop_docker_container()
|
|
|
- logger.info('Container stopped')
|
|
|
- except docker.errors.DockerException as e:
|
|
|
- logger.exception('Failed to stop container', exc_info=False)
|
|
|
- raise e
|
|
|
-
|
|
|
- try:
|
|
|
- # start the container
|
|
|
- mount_dir = config.workspace_mount_path
|
|
|
- self.container = self.docker_client.containers.run(
|
|
|
- self.container_image,
|
|
|
- command='tail -f /dev/null',
|
|
|
- network_mode='host',
|
|
|
- working_dir=self.sandbox_workspace_dir,
|
|
|
- name=self.container_name,
|
|
|
- detach=True,
|
|
|
- volumes={mount_dir: {'bind': self.sandbox_workspace_dir, 'mode': 'rw'}},
|
|
|
- )
|
|
|
- logger.info('Container started')
|
|
|
- except Exception as ex:
|
|
|
- logger.exception('Failed to start container', exc_info=False)
|
|
|
- raise ex
|
|
|
-
|
|
|
- # wait for container to be ready
|
|
|
- elapsed = 0
|
|
|
- while self.container.status != 'running':
|
|
|
- if self.container.status == 'exited':
|
|
|
- logger.info('container exited')
|
|
|
- logger.info('container logs:')
|
|
|
- logger.info(self.container.logs())
|
|
|
- break
|
|
|
- time.sleep(1)
|
|
|
- elapsed += 1
|
|
|
- self.container = self.docker_client.containers.get(self.container_name)
|
|
|
- if elapsed > self.timeout:
|
|
|
- break
|
|
|
- if self.container.status != 'running':
|
|
|
- raise Exception('Failed to start container')
|
|
|
-
|
|
|
- # clean up the container, cannot do it in __del__ because the python interpreter is already shutting down
|
|
|
- def close(self):
|
|
|
- containers = self.docker_client.containers.list(all=True)
|
|
|
- for container in containers:
|
|
|
- try:
|
|
|
- if container.name.startswith(self.container_name_prefix):
|
|
|
- container.remove(force=True)
|
|
|
- except docker.errors.NotFound:
|
|
|
- pass
|
|
|
- self.docker_client.close()
|
|
|
-
|
|
|
- def get_working_directory(self):
|
|
|
- return self.sandbox_workspace_dir
|
|
|
-
|
|
|
- @property
|
|
|
- def user_id(self):
|
|
|
- return config.sandbox_user_id
|
|
|
-
|
|
|
- @property
|
|
|
- def run_as_devin(self):
|
|
|
- # FIXME: On some containers, the devin user doesn't have enough permission, e.g. to install packages
|
|
|
- # How do we make this more flexible?
|
|
|
- return config.run_as_devin
|
|
|
-
|
|
|
- @property
|
|
|
- def sandbox_workspace_dir(self):
|
|
|
- return config.workspace_mount_path_in_sandbox
|
|
|
-
|
|
|
-
|
|
|
-if __name__ == '__main__':
|
|
|
- try:
|
|
|
- exec_box = DockerExecBox()
|
|
|
- except Exception as e:
|
|
|
- logger.exception('Failed to start Docker container: %s', e)
|
|
|
- sys.exit(1)
|
|
|
-
|
|
|
- logger.info(
|
|
|
- "Interactive Docker container started. Type 'exit' or use Ctrl+C to exit."
|
|
|
- )
|
|
|
-
|
|
|
- bg_cmd = exec_box.execute_in_background(
|
|
|
- "while true; do echo -n '.' && sleep 1; done"
|
|
|
- )
|
|
|
-
|
|
|
- sys.stdout.flush()
|
|
|
- try:
|
|
|
- while True:
|
|
|
- try:
|
|
|
- user_input = input('>>> ')
|
|
|
- except EOFError:
|
|
|
- logger.info('Exiting...')
|
|
|
- break
|
|
|
- if user_input.lower() == 'exit':
|
|
|
- logger.info('Exiting...')
|
|
|
- break
|
|
|
- if user_input.lower() == 'kill':
|
|
|
- exec_box.kill_background(bg_cmd.pid)
|
|
|
- logger.info('Background process killed')
|
|
|
- continue
|
|
|
- exit_code, output = exec_box.execute(user_input)
|
|
|
- logger.info('exit code: %d', exit_code)
|
|
|
- logger.info(output)
|
|
|
- if bg_cmd.pid in exec_box.background_commands:
|
|
|
- logs = exec_box.read_logs(bg_cmd.pid)
|
|
|
- logger.info('background logs: %s', logs)
|
|
|
- sys.stdout.flush()
|
|
|
- except KeyboardInterrupt:
|
|
|
- logger.info('Exiting...')
|
|
|
- exec_box.close()
|