add github action for project build on macos and linux (#838)

* update github action to build and run tests  on macos and linux

* fix docker installation

* Fix poetry installation on macos

* Fix docker installation

* Fix docker installation - start docker daemon

* Change docker installation macos

* Update docker buildx version

* new docker installation

* Add new start docker structure

* Add new start docker structure 2

* update github action to build and run tests  on macos and linux

* Update makefile to fix chroma-hnswlib issue with macos

* fix macos build

* Fix macos issue

* Fix macos

* Reformat Makefile

* updates

.github/workflows/build_run-tests.yml

@@ -3,16 +3,45 @@ name: Build & Run Tests
 on: [push, pull_request]
 
 jobs:
-  test:
-    runs-on: ubuntu-latest
+  on-macos:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.11'
-      - name: Run tests
+          python-version: ${{ matrix.python-version }}
+      - name: Install & Start Docker
         run: |
-          curl -sSL https://install.python-poetry.org | python3 -
-          make build
-          poetry run pytest ./tests
+          brew install colima docker
+          colima start
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+      - name: Build Environment
+        run: make build
+      - name: Run Tests
+        run: poetry run pytest ./tests
+  on-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        run: curl -sSL https://install.python-poetry.org | python3 -
+      - name: Build Environment
+        run: make build
+      - name: Run Tests
+        run: poetry run pytest ./tests

Makefile

@@ -20,6 +20,22 @@ RESET=\033[0m
 # Build
 build:
 	@echo "$(GREEN)Building project...$(RESET)"
+	@$(MAKE) -s check-dependencies
+	@$(MAKE) -s pull-docker-image
+	@$(MAKE) -s install-python-dependencies
+	@$(MAKE) -s install-frontend-dependencies
+	@$(MAKE) -s install-precommit-hooks
+	@echo "$(GREEN)Build completed successfully.$(RESET)"
+
+check-dependencies:
+	@echo "$(YELLOW)Checking dependencies...$(RESET)"
+	@$(MAKE) -s check-python
+	@$(MAKE) -s check-npm
+	@$(MAKE) -s check-docker
+	@$(MAKE) -s check-poetry
+	@echo "$(GREEN)Dependencies checked successfully.$(RESET)"
+
+check-python:
 	@echo "$(YELLOW)Checking Python installation...$(RESET)"
 	@if command -v python3 > /dev/null; then \
 		echo "$(BLUE)$(shell python3 --version) is already installed.$(RESET)"; \
@@ -27,6 +43,8 @@ build:
 		echo "$(RED)Python 3 is not installed. Please install Python 3 to continue.$(RESET)"; \
 		exit 1; \
 	fi
+
+check-npm:
 	@echo "$(YELLOW)Checking npm installation...$(RESET)"
 	@if command -v npm > /dev/null; then \
 		echo "$(BLUE)npm $(shell npm --version) is already installed.$(RESET)"; \
@@ -34,6 +52,8 @@ build:
 		echo "$(RED)npm is not installed. Please install Node.js to continue.$(RESET)"; \
 		exit 1; \
 	fi
+
+check-docker:
 	@echo "$(YELLOW)Checking Docker installation...$(RESET)"
 	@if command -v docker > /dev/null; then \
 		echo "$(BLUE)$(shell docker --version) is already installed.$(RESET)"; \
@@ -41,20 +61,35 @@ build:
 		echo "$(RED)Docker is not installed. Please install Docker to continue.$(RESET)"; \
 		exit 1; \
 	fi
-	@echo "$(GREEN)Installing Python dependencies...$(RESET)"
+
+check-poetry:
+	@echo "$(YELLOW)Checking Poetry installation...$(RESET)"
 	@if command -v poetry > /dev/null; then \
-		echo "$(BLUE)Poetry is already installed.$(RESET)"; \
+		echo "$(BLUE)$(shell poetry --version) is already installed.$(RESET)"; \
 	else \
-		echo "$(YELLOW)Poetry is not installed. You can install poetry by running the following command, then adding Poetry to your PATH:"; \
-		echo "$(YELLOW) curl -sSL https://install.python-poetry.org | python3 -$(RESET)"; \
-		echo "$(YELLOW)More detail here: https://python-poetry.org/docs/#installing-with-the-official-installer$(RESET)"; \
+		echo "$(RED)Poetry is not installed. You can install poetry by running the following command, then adding Poetry to your PATH:"; \
+		echo "$(RED) curl -sSL https://install.python-poetry.org | python3 -$(RESET)"; \
+		echo "$(RED)More detail here: https://python-poetry.org/docs/#installing-with-the-official-installer$(RESET)"; \
 		exit 1; \
 	fi
-	@echo "$(GREEN)Pulling Docker image...$(RESET)"
+
+pull-docker-image:
+	@echo "$(YELLOW)Pulling Docker image...$(RESET)"
 	@docker pull $(DOCKER_IMAGE)
+	@echo "$(GREEN)Docker image pulled successfully.$(RESET)"
+
+install-python-dependencies:
+	@echo "$(GREEN)Installing Python dependencies...$(RESET)"
+	@if [ "$(shell uname)" = "Darwin" ]; then \
+		echo "$(BLUE)Installing `chroma-hnswlib`...$(RESET)"; \
+		export HNSWLIB_NO_NATIVE=1; \
+		poetry run pip install chroma-hnswlib; \
+	fi
 	@poetry install --without evaluation
-	@echo "$(GREEN)Activating Poetry shell...$(RESET)"
-	@echo "$(GREEN)Setting up frontend environment...$(RESET)"
+	@echo "$(GREEN)Python dependencies installed successfully.$(RESET)"
+
+install-frontend-dependencies:
+	@echo "$(YELLOW)Setting up frontend environment...$(RESET)"
 	@echo "$(YELLOW)Detect Node.js version...$(RESET)"
 	@cd frontend && node ./scripts/detect-node-version.js
 	@cd frontend && \
@@ -62,29 +97,33 @@ build:
 		npm install && \
 		echo "$(BLUE)Running make-i18n with npm...$(RESET)" && \
 		npm run make-i18n
-	@echo "$(GREEN)Installing pre-commit hooks...$(RESET)"
+	@echo "$(GREEN)Frontend dependencies installed successfully.$(RESET)"
+
+install-precommit-hooks:
+	@echo "$(YELLOW)Installing pre-commit hooks...$(RESET)"
 	@git config --unset-all core.hooksPath || true
 	@poetry run pre-commit install --config $(PRECOMMIT_CONFIG_PATH)
-	@echo "$(GREEN)Build completed successfully.$(RESET)"
+	@echo "$(GREEN)Pre-commit hooks installed successfully.$(RESET)"
 
 # Start backend
 start-backend:
-	@echo "$(GREEN)Starting backend...$(RESET)"
+	@echo "$(YELLOW)Starting backend...$(RESET)"
 	@poetry run uvicorn opendevin.server.listen:app --port $(BACKEND_PORT)
 
 # Start frontend
 start-frontend:
-	@echo "$(GREEN)Starting frontend...$(RESET)"
+	@echo "$(YELLOW)Starting frontend...$(RESET)"
 	@cd frontend && BACKEND_HOST=$(BACKEND_HOST) FRONTEND_PORT=$(FRONTEND_PORT) npm run start
 
 # Run the app
 run:
-	@echo "$(GREEN)Running the app...$(RESET)"
+	@echo "$(YELLOW)Running the app...$(RESET)"
 	@if [ "$(OS)" = "Windows_NT" ]; then \
 		echo "$(RED)`make run` is not supported on Windows. Please run `make start-frontend` and `make start-backend` separately.$(RESET)"; \
 		exit 1; \
 	fi
 	@mkdir -p logs
+	@echo "$(YELLOW)Starting backend server...$(RESET)"
 	@poetry run nohup uvicorn opendevin.server.listen:app --port $(BACKEND_PORT) > logs/backend_$(shell date +'%Y%m%d_%H%M%S').log 2>&1 &
 	@echo "$(YELLOW)Waiting for the backend to start...$(RESET)"
 	@until nc -z localhost $(BACKEND_PORT); do sleep 0.1; done
@@ -94,7 +133,12 @@ run:
 
 # Setup config.toml
 setup-config:
-	@echo "$(GREEN)Setting up config.toml...$(RESET)"
+	@echo "$(YELLOW)Setting up config.toml...$(RESET)"
+	@$(MAKE) setup-config-prompts
+	@mv $(CONFIG_FILE).tmp $(CONFIG_FILE)
+	@echo "$(GREEN)Config.toml setup completed.$(RESET)"
+
+setup-config-prompts:
 	@read -p "Enter your LLM Model name (see https://docs.litellm.ai/docs/providers for full list) [default: $(DEFAULT_MODEL)]: " llm_model; \
 	 llm_model=$${llm_model:-$(DEFAULT_MODEL)}; \
 	 echo "LLM_MODEL=\"$$llm_model\"" > $(CONFIG_FILE).tmp
@@ -124,22 +168,18 @@ setup-config:
 	 workspace_dir=$${workspace_dir:-$(DEFAULT_WORKSPACE_DIR)}; \
 	 echo "WORKSPACE_DIR=\"$$workspace_dir\"" >> $(CONFIG_FILE).tmp
 
-	@mv $(CONFIG_FILE).tmp $(CONFIG_FILE)
-	@echo "$(GREEN)Config.toml setup completed.$(RESET)"
-
 # Help
 help:
 	@echo "$(BLUE)Usage: make [target]$(RESET)"
 	@echo "Targets:"
 	@echo "  $(GREEN)build$(RESET)               - Build project, including environment setup and dependencies."
-	@echo "  $(GREEN)build-eval$(RESET)          - Build project evaluation pipeline, including environment setup and dependencies."
+	@echo "  $(GREEN)setup-config$(RESET)        - Setup the configuration for OpenDevin by providing LLM API key,"
+	@echo "                        LLM Model name, and workspace directory."
 	@echo "  $(GREEN)start-backend$(RESET)       - Start the backend server for the OpenDevin project."
 	@echo "  $(GREEN)start-frontend$(RESET)      - Start the frontend server for the OpenDevin project."
 	@echo "  $(GREEN)run$(RESET)                 - Run the OpenDevin application, starting both backend and frontend servers."
 	@echo "                        Backend Log file will be stored in the 'logs' directory."
-	@echo "  $(GREEN)setup-config$(RESET)        - Setup the configuration for OpenDevin by providing LLM API key,"
-	@echo "                        LLM Model name, and workspace directory."
 	@echo "  $(GREEN)help$(RESET)                - Display this help message, providing information on available targets."
 
 # Phony targets
-.PHONY: build build-eval start-backend start-frontend run setup-config help
+.PHONY: build check-dependencies check-python check-npm check-docker check-poetry pull-docker-image install-python-dependencies install-frontend-dependencies install-precommit-hooks start-backend start-frontend run setup-config setup-config-prompts help

poetry.lock

@@ -1792,13 +1792,13 @@ adal = ["adal (>=1.0.2)"]
 
 [[package]]
 name = "litellm"
-version = "1.34.28"
+version = "1.34.33"
 description = "Library to easily interface with LLM API providers"
 optional = false
 python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
 files = [
-    {file = "litellm-1.34.28-py3-none-any.whl", hash = "sha256:1e175b5a3e07b133535a3c42b1249cd16db04b820a2adedfa7fdc5badec8eaef"},
-    {file = "litellm-1.34.28.tar.gz", hash = "sha256:734e193d6567a8a41db859d0053386ca234821e689c7ba921a3604d766b5a90a"},
+    {file = "litellm-1.34.33-py3-none-any.whl", hash = "sha256:88164642dcc30239c9294f1375c7b15c4f664f927b9e5b6d7c81936e67daeec7"},
+    {file = "litellm-1.34.33.tar.gz", hash = "sha256:43b4b06086aa934131e056f46cc553e160abc63dd40a844b9bae65a603914557"},
 ]
 
 [package.dependencies]
@@ -2725,20 +2725,20 @@ files = [
 
 [[package]]
 name = "networkx"
-version = "3.2.1"
+version = "3.3"
 description = "Python package for creating and manipulating graphs and networks"
 optional = false
-python-versions = ">=3.9"
+python-versions = ">=3.10"
 files = [
-    {file = "networkx-3.2.1-py3-none-any.whl", hash = "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2"},
-    {file = "networkx-3.2.1.tar.gz", hash = "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6"},
+    {file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"},
+    {file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"},
 ]
 
 [package.extras]
-default = ["matplotlib (>=3.5)", "numpy (>=1.22)", "pandas (>=1.4)", "scipy (>=1.9,!=1.11.0,!=1.11.1)"]
-developer = ["changelist (==0.4)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"]
-doc = ["nb2plots (>=0.7)", "nbconvert (<7.9)", "numpydoc (>=1.6)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.14)", "sphinx (>=7)", "sphinx-gallery (>=0.14)", "texext (>=0.6.7)"]
-extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.11)", "sympy (>=1.10)"]
+default = ["matplotlib (>=3.6)", "numpy (>=1.23)", "pandas (>=1.4)", "scipy (>=1.9,!=1.11.0,!=1.11.1)"]
+developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"]
+doc = ["myst-nb (>=1.0)", "numpydoc (>=1.7)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.14)", "sphinx (>=7)", "sphinx-gallery (>=0.14)", "texext (>=0.6.7)"]
+extra = ["lxml (>=4.6)", "pydot (>=2.0)", "pygraphviz (>=1.12)", "sympy (>=1.10)"]
 test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"]
 
 [[package]]
@@ -5874,4 +5874,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11"
-content-hash = "22c64f0b7b063c1da60fb98a76b1827c75edea6b35de17b73f0bc72daea47d69"
+content-hash = "eb7d77f58c52f70702e9a8501084b09c307d62caf179428b70b781860508a0fb"

pyproject.toml

@@ -31,7 +31,6 @@ chromadb = "*"
 llama-index-embeddings-huggingface = "*"
 llama-index-embeddings-azure-openai = "*"
 llama-index-embeddings-ollama = "*"
-pymupdfb = "*"
 
 [tool.poetry.group.dev.dependencies]
 ruff = "*"