test_send_pull_request.py 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276
  1. import os
  2. import tempfile
  3. from unittest.mock import MagicMock, call, patch
  4. import pytest
  5. from openhands.core.config import LLMConfig
  6. from openhands.resolver.github_issue import ReviewThread
  7. from openhands.resolver.resolver_output import GithubIssue, ResolverOutput
  8. from openhands.resolver.send_pull_request import (
  9. apply_patch,
  10. initialize_repo,
  11. load_single_resolver_output,
  12. make_commit,
  13. process_all_successful_issues,
  14. process_single_issue,
  15. reply_to_comment,
  16. send_pull_request,
  17. update_existing_pull_request,
  18. )
  19. @pytest.fixture
  20. def mock_output_dir():
  21. with tempfile.TemporaryDirectory() as temp_dir:
  22. repo_path = os.path.join(temp_dir, 'repo')
  23. # Initialize a GitHub repo in "repo" and add a commit with "README.md"
  24. os.makedirs(repo_path)
  25. os.system(f'git init {repo_path}')
  26. readme_path = os.path.join(repo_path, 'README.md')
  27. with open(readme_path, 'w') as f:
  28. f.write('hello world')
  29. os.system(f'git -C {repo_path} add README.md')
  30. os.system(f"git -C {repo_path} commit -m 'Initial commit'")
  31. yield temp_dir
  32. @pytest.fixture
  33. def mock_github_issue():
  34. return GithubIssue(
  35. number=42,
  36. title='Test Issue',
  37. owner='test-owner',
  38. repo='test-repo',
  39. body='Test body',
  40. )
  41. @pytest.fixture
  42. def mock_llm_config():
  43. return LLMConfig()
  44. def test_load_single_resolver_output():
  45. mock_output_jsonl = 'tests/unit/resolver/mock_output/output.jsonl'
  46. # Test loading an existing issue
  47. resolver_output = load_single_resolver_output(mock_output_jsonl, 5)
  48. assert isinstance(resolver_output, ResolverOutput)
  49. assert resolver_output.issue.number == 5
  50. assert resolver_output.issue.title == 'Add MIT license'
  51. assert resolver_output.issue.owner == 'neubig'
  52. assert resolver_output.issue.repo == 'pr-viewer'
  53. # Test loading a non-existent issue
  54. with pytest.raises(ValueError):
  55. load_single_resolver_output(mock_output_jsonl, 999)
  56. def test_apply_patch(mock_output_dir):
  57. # Create a sample file in the mock repo
  58. sample_file = os.path.join(mock_output_dir, 'sample.txt')
  59. with open(sample_file, 'w') as f:
  60. f.write('Original content')
  61. # Create a sample patch
  62. patch_content = """
  63. diff --git a/sample.txt b/sample.txt
  64. index 9daeafb..b02def2 100644
  65. --- a/sample.txt
  66. +++ b/sample.txt
  67. @@ -1 +1,2 @@
  68. -Original content
  69. +Updated content
  70. +New line
  71. """
  72. # Apply the patch
  73. apply_patch(mock_output_dir, patch_content)
  74. # Check if the file was updated correctly
  75. with open(sample_file, 'r') as f:
  76. updated_content = f.read()
  77. assert updated_content.strip() == 'Updated content\nNew line'.strip()
  78. def test_apply_patch_preserves_line_endings(mock_output_dir):
  79. # Create sample files with different line endings
  80. unix_file = os.path.join(mock_output_dir, 'unix_style.txt')
  81. dos_file = os.path.join(mock_output_dir, 'dos_style.txt')
  82. with open(unix_file, 'w', newline='\n') as f:
  83. f.write('Line 1\nLine 2\nLine 3')
  84. with open(dos_file, 'w', newline='\r\n') as f:
  85. f.write('Line 1\r\nLine 2\r\nLine 3')
  86. # Create patches for both files
  87. unix_patch = """
  88. diff --git a/unix_style.txt b/unix_style.txt
  89. index 9daeafb..b02def2 100644
  90. --- a/unix_style.txt
  91. +++ b/unix_style.txt
  92. @@ -1,3 +1,3 @@
  93. Line 1
  94. -Line 2
  95. +Updated Line 2
  96. Line 3
  97. """
  98. dos_patch = """
  99. diff --git a/dos_style.txt b/dos_style.txt
  100. index 9daeafb..b02def2 100644
  101. --- a/dos_style.txt
  102. +++ b/dos_style.txt
  103. @@ -1,3 +1,3 @@
  104. Line 1
  105. -Line 2
  106. +Updated Line 2
  107. Line 3
  108. """
  109. # Apply patches
  110. apply_patch(mock_output_dir, unix_patch)
  111. apply_patch(mock_output_dir, dos_patch)
  112. # Check if line endings are preserved
  113. with open(unix_file, 'rb') as f:
  114. unix_content = f.read()
  115. with open(dos_file, 'rb') as f:
  116. dos_content = f.read()
  117. assert (
  118. b'\r\n' not in unix_content
  119. ), 'Unix-style line endings were changed to DOS-style'
  120. assert b'\r\n' in dos_content, 'DOS-style line endings were changed to Unix-style'
  121. # Check if content was updated correctly
  122. assert unix_content.decode('utf-8').split('\n')[1] == 'Updated Line 2'
  123. assert dos_content.decode('utf-8').split('\r\n')[1] == 'Updated Line 2'
  124. def test_apply_patch_create_new_file(mock_output_dir):
  125. # Create a patch that adds a new file
  126. patch_content = """
  127. diff --git a/new_file.txt b/new_file.txt
  128. new file mode 100644
  129. index 0000000..3b18e51
  130. --- /dev/null
  131. +++ b/new_file.txt
  132. @@ -0,0 +1 @@
  133. +hello world
  134. """
  135. # Apply the patch
  136. apply_patch(mock_output_dir, patch_content)
  137. # Check if the new file was created
  138. new_file_path = os.path.join(mock_output_dir, 'new_file.txt')
  139. assert os.path.exists(new_file_path), 'New file was not created'
  140. # Check if the file content is correct
  141. with open(new_file_path, 'r') as f:
  142. content = f.read().strip()
  143. assert content == 'hello world', 'File content is incorrect'
  144. def test_apply_patch_rename_file(mock_output_dir):
  145. # Create a sample file in the mock repo
  146. old_file = os.path.join(mock_output_dir, 'old_name.txt')
  147. with open(old_file, 'w') as f:
  148. f.write('This file will be renamed')
  149. # Create a patch that renames the file
  150. patch_content = """diff --git a/old_name.txt b/new_name.txt
  151. similarity index 100%
  152. rename from old_name.txt
  153. rename to new_name.txt"""
  154. # Apply the patch
  155. apply_patch(mock_output_dir, patch_content)
  156. # Check if the file was renamed
  157. new_file = os.path.join(mock_output_dir, 'new_name.txt')
  158. assert not os.path.exists(old_file), 'Old file still exists'
  159. assert os.path.exists(new_file), 'New file was not created'
  160. # Check if the content is preserved
  161. with open(new_file, 'r') as f:
  162. content = f.read()
  163. assert content == 'This file will be renamed'
  164. def test_apply_patch_delete_file(mock_output_dir):
  165. # Create a sample file in the mock repo
  166. sample_file = os.path.join(mock_output_dir, 'to_be_deleted.txt')
  167. with open(sample_file, 'w') as f:
  168. f.write('This file will be deleted')
  169. # Create a patch that deletes the file
  170. patch_content = """
  171. diff --git a/to_be_deleted.txt b/to_be_deleted.txt
  172. deleted file mode 100644
  173. index 9daeafb..0000000
  174. --- a/to_be_deleted.txt
  175. +++ /dev/null
  176. @@ -1 +0,0 @@
  177. -This file will be deleted
  178. """
  179. # Apply the patch
  180. apply_patch(mock_output_dir, patch_content)
  181. # Check if the file was deleted
  182. assert not os.path.exists(sample_file), 'File was not deleted'
  183. def test_initialize_repo(mock_output_dir):
  184. issue_type = 'issue'
  185. # Copy the repo to patches
  186. ISSUE_NUMBER = 3
  187. initialize_repo(mock_output_dir, ISSUE_NUMBER, issue_type)
  188. patches_dir = os.path.join(mock_output_dir, 'patches', f'issue_{ISSUE_NUMBER}')
  189. # Check if files were copied correctly
  190. assert os.path.exists(os.path.join(patches_dir, 'README.md'))
  191. # Check file contents
  192. with open(os.path.join(patches_dir, 'README.md'), 'r') as f:
  193. assert f.read() == 'hello world'
  194. @patch('openhands.resolver.send_pull_request.reply_to_comment')
  195. @patch('requests.post')
  196. @patch('subprocess.run')
  197. @patch('openhands.resolver.send_pull_request.LLM')
  198. def test_update_existing_pull_request(
  199. mock_llm_class,
  200. mock_subprocess_run,
  201. mock_requests_post,
  202. mock_reply_to_comment,
  203. ):
  204. # Arrange: Set up test data
  205. github_issue = GithubIssue(
  206. owner='test-owner',
  207. repo='test-repo',
  208. number=1,
  209. title='Test PR',
  210. body='This is a test PR',
  211. thread_ids=['comment1', 'comment2'],
  212. head_branch='test-branch',
  213. )
  214. github_token = 'test-token'
  215. github_username = 'test-user'
  216. patch_dir = '/path/to/patch'
  217. additional_message = '["Fixed bug in function A", "Updated documentation for B"]'
  218. # Mock the subprocess.run call for git push
  219. mock_subprocess_run.return_value = MagicMock(returncode=0)
  220. # Mock the requests.post call for adding a PR comment
  221. mock_requests_post.return_value.status_code = 201
  222. # Mock LLM instance and completion call
  223. mock_llm_instance = MagicMock()
  224. mock_completion_response = MagicMock()
  225. mock_completion_response.choices = [
  226. MagicMock(message=MagicMock(content='This is an issue resolution.'))
  227. ]
  228. mock_llm_instance.completion.return_value = mock_completion_response
  229. mock_llm_class.return_value = mock_llm_instance
  230. llm_config = LLMConfig()
  231. # Act: Call the function without comment_message to test auto-generation
  232. result = update_existing_pull_request(
  233. github_issue,
  234. github_token,
  235. github_username,
  236. patch_dir,
  237. llm_config,
  238. comment_message=None,
  239. additional_message=additional_message,
  240. )
  241. # Assert: Check if the git push command was executed
  242. push_command = (
  243. f'git -C {patch_dir} push '
  244. f'https://{github_username}:{github_token}@github.com/'
  245. f'{github_issue.owner}/{github_issue.repo}.git {github_issue.head_branch}'
  246. )
  247. mock_subprocess_run.assert_called_once_with(
  248. push_command, shell=True, capture_output=True, text=True
  249. )
  250. # Assert: Check if the auto-generated comment was posted to the PR
  251. comment_url = f'https://api.github.com/repos/{github_issue.owner}/{github_issue.repo}/issues/{github_issue.number}/comments'
  252. expected_comment = 'This is an issue resolution.'
  253. mock_requests_post.assert_called_once_with(
  254. comment_url,
  255. headers={
  256. 'Authorization': f'token {github_token}',
  257. 'Accept': 'application/vnd.github.v3+json',
  258. },
  259. json={'body': expected_comment},
  260. )
  261. # Assert: Check if the reply_to_comment function was called for each thread ID
  262. mock_reply_to_comment.assert_has_calls(
  263. [
  264. call(github_token, 'comment1', 'Fixed bug in function A'),
  265. call(github_token, 'comment2', 'Updated documentation for B'),
  266. ]
  267. )
  268. # Assert: Check the returned PR URL
  269. assert (
  270. result
  271. == f'https://github.com/{github_issue.owner}/{github_issue.repo}/pull/{github_issue.number}'
  272. )
  273. @pytest.mark.parametrize(
  274. 'pr_type,target_branch',
  275. [
  276. ('branch', None),
  277. ('draft', None),
  278. ('ready', None),
  279. ('branch', 'feature'),
  280. ('draft', 'develop'),
  281. ('ready', 'staging'),
  282. ],
  283. )
  284. @patch('subprocess.run')
  285. @patch('requests.post')
  286. @patch('requests.get')
  287. def test_send_pull_request(
  288. mock_get,
  289. mock_post,
  290. mock_run,
  291. mock_github_issue,
  292. mock_output_dir,
  293. pr_type,
  294. target_branch,
  295. ):
  296. repo_path = os.path.join(mock_output_dir, 'repo')
  297. # Mock API responses based on whether target_branch is specified
  298. if target_branch:
  299. mock_get.side_effect = [
  300. MagicMock(status_code=404), # Branch doesn't exist
  301. MagicMock(status_code=200), # Target branch exists
  302. ]
  303. else:
  304. mock_get.side_effect = [
  305. MagicMock(status_code=404), # Branch doesn't exist
  306. MagicMock(json=lambda: {'default_branch': 'main'}), # Get default branch
  307. ]
  308. mock_post.return_value.json.return_value = {
  309. 'html_url': 'https://github.com/test-owner/test-repo/pull/1'
  310. }
  311. # Mock subprocess.run calls
  312. mock_run.side_effect = [
  313. MagicMock(returncode=0), # git checkout -b
  314. MagicMock(returncode=0), # git push
  315. ]
  316. # Call the function
  317. result = send_pull_request(
  318. github_issue=mock_github_issue,
  319. github_token='test-token',
  320. github_username='test-user',
  321. patch_dir=repo_path,
  322. pr_type=pr_type,
  323. target_branch=target_branch,
  324. )
  325. # Assert API calls
  326. expected_get_calls = 2
  327. assert mock_get.call_count == expected_get_calls
  328. # Check branch creation and push
  329. assert mock_run.call_count == 2
  330. checkout_call, push_call = mock_run.call_args_list
  331. assert checkout_call == call(
  332. ['git', '-C', repo_path, 'checkout', '-b', 'openhands-fix-issue-42'],
  333. capture_output=True,
  334. text=True,
  335. )
  336. assert push_call == call(
  337. [
  338. 'git',
  339. '-C',
  340. repo_path,
  341. 'push',
  342. 'https://test-user:test-token@github.com/test-owner/test-repo.git',
  343. 'openhands-fix-issue-42',
  344. ],
  345. capture_output=True,
  346. text=True,
  347. )
  348. # Check PR creation based on pr_type
  349. if pr_type == 'branch':
  350. assert (
  351. result
  352. == 'https://github.com/test-owner/test-repo/compare/openhands-fix-issue-42?expand=1'
  353. )
  354. mock_post.assert_not_called()
  355. else:
  356. assert result == 'https://github.com/test-owner/test-repo/pull/1'
  357. mock_post.assert_called_once()
  358. post_data = mock_post.call_args[1]['json']
  359. assert post_data['title'] == 'Fix issue #42: Test Issue'
  360. assert post_data['body'].startswith('This pull request fixes #42.')
  361. assert post_data['head'] == 'openhands-fix-issue-42'
  362. assert post_data['base'] == (target_branch if target_branch else 'main')
  363. assert post_data['draft'] == (pr_type == 'draft')
  364. @patch('subprocess.run')
  365. @patch('requests.post')
  366. @patch('requests.get')
  367. def test_send_pull_request_with_reviewer(
  368. mock_get, mock_post, mock_run, mock_github_issue, mock_output_dir
  369. ):
  370. repo_path = os.path.join(mock_output_dir, 'repo')
  371. reviewer = 'test-reviewer'
  372. # Mock API responses
  373. mock_get.side_effect = [
  374. MagicMock(status_code=404), # Branch doesn't exist
  375. MagicMock(json=lambda: {'default_branch': 'main'}), # Get default branch
  376. ]
  377. # Mock PR creation response
  378. mock_post.side_effect = [
  379. MagicMock(
  380. status_code=201,
  381. json=lambda: {
  382. 'html_url': 'https://github.com/test-owner/test-repo/pull/1',
  383. 'number': 1,
  384. },
  385. ), # PR creation
  386. MagicMock(status_code=201), # Reviewer request
  387. ]
  388. # Mock subprocess.run calls
  389. mock_run.side_effect = [
  390. MagicMock(returncode=0), # git checkout -b
  391. MagicMock(returncode=0), # git push
  392. ]
  393. # Call the function with reviewer
  394. result = send_pull_request(
  395. github_issue=mock_github_issue,
  396. github_token='test-token',
  397. github_username='test-user',
  398. patch_dir=repo_path,
  399. pr_type='ready',
  400. reviewer=reviewer,
  401. )
  402. # Assert API calls
  403. assert mock_get.call_count == 2
  404. assert mock_post.call_count == 2
  405. # Check PR creation
  406. pr_create_call = mock_post.call_args_list[0]
  407. assert pr_create_call[1]['json']['title'] == 'Fix issue #42: Test Issue'
  408. # Check reviewer request
  409. reviewer_request_call = mock_post.call_args_list[1]
  410. assert (
  411. reviewer_request_call[0][0]
  412. == 'https://api.github.com/repos/test-owner/test-repo/pulls/1/requested_reviewers'
  413. )
  414. assert reviewer_request_call[1]['json'] == {'reviewers': ['test-reviewer']}
  415. # Check the result URL
  416. assert result == 'https://github.com/test-owner/test-repo/pull/1'
  417. @patch('requests.get')
  418. def test_send_pull_request_invalid_target_branch(
  419. mock_get, mock_github_issue, mock_output_dir
  420. ):
  421. """Test that an error is raised when specifying a non-existent target branch"""
  422. repo_path = os.path.join(mock_output_dir, 'repo')
  423. # Mock API response for non-existent branch
  424. mock_get.side_effect = [
  425. MagicMock(status_code=404), # Branch doesn't exist
  426. MagicMock(status_code=404), # Target branch doesn't exist
  427. ]
  428. # Test that ValueError is raised when target branch doesn't exist
  429. with pytest.raises(
  430. ValueError, match='Target branch nonexistent-branch does not exist'
  431. ):
  432. send_pull_request(
  433. github_issue=mock_github_issue,
  434. github_token='test-token',
  435. github_username='test-user',
  436. patch_dir=repo_path,
  437. pr_type='ready',
  438. target_branch='nonexistent-branch',
  439. )
  440. # Verify API calls
  441. assert mock_get.call_count == 2
  442. @patch('subprocess.run')
  443. @patch('requests.post')
  444. @patch('requests.get')
  445. def test_send_pull_request_git_push_failure(
  446. mock_get, mock_post, mock_run, mock_github_issue, mock_output_dir
  447. ):
  448. repo_path = os.path.join(mock_output_dir, 'repo')
  449. # Mock API responses
  450. mock_get.return_value = MagicMock(json=lambda: {'default_branch': 'main'})
  451. # Mock the subprocess.run calls
  452. mock_run.side_effect = [
  453. MagicMock(returncode=0), # git checkout -b
  454. MagicMock(returncode=1, stderr='Error: failed to push some refs'), # git push
  455. ]
  456. # Test that RuntimeError is raised when git push fails
  457. with pytest.raises(
  458. RuntimeError, match='Failed to push changes to the remote repository'
  459. ):
  460. send_pull_request(
  461. github_issue=mock_github_issue,
  462. github_token='test-token',
  463. github_username='test-user',
  464. patch_dir=repo_path,
  465. pr_type='ready',
  466. )
  467. # Assert that subprocess.run was called twice
  468. assert mock_run.call_count == 2
  469. # Check the git checkout -b command
  470. checkout_call = mock_run.call_args_list[0]
  471. assert checkout_call[0][0] == [
  472. 'git',
  473. '-C',
  474. repo_path,
  475. 'checkout',
  476. '-b',
  477. 'openhands-fix-issue-42',
  478. ]
  479. # Check the git push command
  480. push_call = mock_run.call_args_list[1]
  481. assert push_call[0][0] == [
  482. 'git',
  483. '-C',
  484. repo_path,
  485. 'push',
  486. 'https://test-user:test-token@github.com/test-owner/test-repo.git',
  487. 'openhands-fix-issue-42',
  488. ]
  489. # Assert that no pull request was created
  490. mock_post.assert_not_called()
  491. @patch('subprocess.run')
  492. @patch('requests.post')
  493. @patch('requests.get')
  494. def test_send_pull_request_permission_error(
  495. mock_get, mock_post, mock_run, mock_github_issue, mock_output_dir
  496. ):
  497. repo_path = os.path.join(mock_output_dir, 'repo')
  498. # Mock API responses
  499. mock_get.return_value = MagicMock(json=lambda: {'default_branch': 'main'})
  500. mock_post.return_value.status_code = 403
  501. # Mock subprocess.run calls
  502. mock_run.side_effect = [
  503. MagicMock(returncode=0), # git checkout -b
  504. MagicMock(returncode=0), # git push
  505. ]
  506. # Test that RuntimeError is raised when PR creation fails due to permissions
  507. with pytest.raises(
  508. RuntimeError, match='Failed to create pull request due to missing permissions.'
  509. ):
  510. send_pull_request(
  511. github_issue=mock_github_issue,
  512. github_token='test-token',
  513. github_username='test-user',
  514. patch_dir=repo_path,
  515. pr_type='ready',
  516. )
  517. # Assert that the branch was created and pushed
  518. assert mock_run.call_count == 2
  519. mock_post.assert_called_once()
  520. @patch('requests.post')
  521. def test_reply_to_comment(mock_post):
  522. # Arrange: set up the test data
  523. github_token = 'test_token'
  524. comment_id = 'test_comment_id'
  525. reply = 'This is a test reply.'
  526. # Mock the response from the GraphQL API
  527. mock_response = MagicMock()
  528. mock_response.status_code = 200
  529. mock_response.json.return_value = {
  530. 'data': {
  531. 'addPullRequestReviewThreadReply': {
  532. 'comment': {
  533. 'id': 'test_reply_id',
  534. 'body': 'Openhands fix success summary\n\n\nThis is a test reply.',
  535. 'createdAt': '2024-10-01T12:34:56Z',
  536. }
  537. }
  538. }
  539. }
  540. mock_post.return_value = mock_response
  541. # Act: call the function
  542. reply_to_comment(github_token, comment_id, reply)
  543. # Assert: check that the POST request was made with the correct parameters
  544. query = """
  545. mutation($body: String!, $pullRequestReviewThreadId: ID!) {
  546. addPullRequestReviewThreadReply(input: { body: $body, pullRequestReviewThreadId: $pullRequestReviewThreadId }) {
  547. comment {
  548. id
  549. body
  550. createdAt
  551. }
  552. }
  553. }
  554. """
  555. expected_variables = {
  556. 'body': 'Openhands fix success summary\n\n\nThis is a test reply.',
  557. 'pullRequestReviewThreadId': comment_id,
  558. }
  559. # Check that the correct request was made to the API
  560. mock_post.assert_called_once_with(
  561. 'https://api.github.com/graphql',
  562. json={'query': query, 'variables': expected_variables},
  563. headers={
  564. 'Authorization': f'Bearer {github_token}',
  565. 'Content-Type': 'application/json',
  566. },
  567. )
  568. # Check that the response status was checked (via response.raise_for_status)
  569. mock_response.raise_for_status.assert_called_once()
  570. @patch('openhands.resolver.send_pull_request.initialize_repo')
  571. @patch('openhands.resolver.send_pull_request.apply_patch')
  572. @patch('openhands.resolver.send_pull_request.update_existing_pull_request')
  573. @patch('openhands.resolver.send_pull_request.make_commit')
  574. def test_process_single_pr_update(
  575. mock_make_commit,
  576. mock_update_existing_pull_request,
  577. mock_apply_patch,
  578. mock_initialize_repo,
  579. mock_output_dir,
  580. mock_llm_config,
  581. ):
  582. # Initialize test data
  583. github_token = 'test_token'
  584. github_username = 'test_user'
  585. pr_type = 'draft'
  586. resolver_output = ResolverOutput(
  587. issue=GithubIssue(
  588. owner='test-owner',
  589. repo='test-repo',
  590. number=1,
  591. title='Issue 1',
  592. body='Body 1',
  593. closing_issues=[],
  594. review_threads=[
  595. ReviewThread(comment='review comment for feedback', files=[])
  596. ],
  597. thread_ids=['1'],
  598. head_branch='branch 1',
  599. ),
  600. issue_type='pr',
  601. instruction='Test instruction 1',
  602. base_commit='def456',
  603. git_patch='Test patch 1',
  604. history=[],
  605. metrics={},
  606. success=True,
  607. comment_success=None,
  608. success_explanation='[Test success 1]',
  609. error=None,
  610. )
  611. mock_update_existing_pull_request.return_value = (
  612. 'https://github.com/test-owner/test-repo/pull/1'
  613. )
  614. mock_initialize_repo.return_value = f'{mock_output_dir}/patches/pr_1'
  615. process_single_issue(
  616. mock_output_dir,
  617. resolver_output,
  618. github_token,
  619. github_username,
  620. pr_type,
  621. mock_llm_config,
  622. None,
  623. False,
  624. None,
  625. )
  626. mock_initialize_repo.assert_called_once_with(mock_output_dir, 1, 'pr', 'branch 1')
  627. mock_apply_patch.assert_called_once_with(
  628. f'{mock_output_dir}/patches/pr_1', resolver_output.git_patch
  629. )
  630. mock_make_commit.assert_called_once_with(
  631. f'{mock_output_dir}/patches/pr_1', resolver_output.issue, 'pr'
  632. )
  633. mock_update_existing_pull_request.assert_called_once_with(
  634. github_issue=resolver_output.issue,
  635. github_token=github_token,
  636. github_username=github_username,
  637. patch_dir=f'{mock_output_dir}/patches/pr_1',
  638. additional_message='[Test success 1]',
  639. llm_config=mock_llm_config,
  640. )
  641. @patch('openhands.resolver.send_pull_request.initialize_repo')
  642. @patch('openhands.resolver.send_pull_request.apply_patch')
  643. @patch('openhands.resolver.send_pull_request.send_pull_request')
  644. @patch('openhands.resolver.send_pull_request.make_commit')
  645. def test_process_single_issue(
  646. mock_make_commit,
  647. mock_send_pull_request,
  648. mock_apply_patch,
  649. mock_initialize_repo,
  650. mock_output_dir,
  651. mock_llm_config,
  652. ):
  653. # Initialize test data
  654. github_token = 'test_token'
  655. github_username = 'test_user'
  656. pr_type = 'draft'
  657. resolver_output = ResolverOutput(
  658. issue=GithubIssue(
  659. owner='test-owner',
  660. repo='test-repo',
  661. number=1,
  662. title='Issue 1',
  663. body='Body 1',
  664. ),
  665. issue_type='issue',
  666. instruction='Test instruction 1',
  667. base_commit='def456',
  668. git_patch='Test patch 1',
  669. history=[],
  670. metrics={},
  671. success=True,
  672. comment_success=None,
  673. success_explanation='Test success 1',
  674. error=None,
  675. )
  676. # Mock return value
  677. mock_send_pull_request.return_value = (
  678. 'https://github.com/test-owner/test-repo/pull/1'
  679. )
  680. mock_initialize_repo.return_value = f'{mock_output_dir}/patches/issue_1'
  681. # Call the function
  682. process_single_issue(
  683. mock_output_dir,
  684. resolver_output,
  685. github_token,
  686. github_username,
  687. pr_type,
  688. mock_llm_config,
  689. None,
  690. False,
  691. None,
  692. )
  693. # Assert that the mocked functions were called with correct arguments
  694. mock_initialize_repo.assert_called_once_with(mock_output_dir, 1, 'issue', 'def456')
  695. mock_apply_patch.assert_called_once_with(
  696. f'{mock_output_dir}/patches/issue_1', resolver_output.git_patch
  697. )
  698. mock_make_commit.assert_called_once_with(
  699. f'{mock_output_dir}/patches/issue_1', resolver_output.issue, 'issue'
  700. )
  701. mock_send_pull_request.assert_called_once_with(
  702. github_issue=resolver_output.issue,
  703. github_token=github_token,
  704. github_username=github_username,
  705. patch_dir=f'{mock_output_dir}/patches/issue_1',
  706. pr_type=pr_type,
  707. fork_owner=None,
  708. additional_message=resolver_output.success_explanation,
  709. target_branch=None,
  710. reviewer=None,
  711. )
  712. @patch('openhands.resolver.send_pull_request.initialize_repo')
  713. @patch('openhands.resolver.send_pull_request.apply_patch')
  714. @patch('openhands.resolver.send_pull_request.send_pull_request')
  715. @patch('openhands.resolver.send_pull_request.make_commit')
  716. def test_process_single_issue_unsuccessful(
  717. mock_make_commit,
  718. mock_send_pull_request,
  719. mock_apply_patch,
  720. mock_initialize_repo,
  721. mock_output_dir,
  722. mock_llm_config,
  723. ):
  724. # Initialize test data
  725. github_token = 'test_token'
  726. github_username = 'test_user'
  727. pr_type = 'draft'
  728. resolver_output = ResolverOutput(
  729. issue=GithubIssue(
  730. owner='test-owner',
  731. repo='test-repo',
  732. number=1,
  733. title='Issue 1',
  734. body='Body 1',
  735. ),
  736. issue_type='issue',
  737. instruction='Test instruction 1',
  738. base_commit='def456',
  739. git_patch='Test patch 1',
  740. history=[],
  741. metrics={},
  742. success=False,
  743. comment_success=None,
  744. success_explanation='',
  745. error='Test error',
  746. )
  747. # Call the function
  748. process_single_issue(
  749. mock_output_dir,
  750. resolver_output,
  751. github_token,
  752. github_username,
  753. pr_type,
  754. mock_llm_config,
  755. None,
  756. False,
  757. None,
  758. )
  759. # Assert that none of the mocked functions were called
  760. mock_initialize_repo.assert_not_called()
  761. mock_apply_patch.assert_not_called()
  762. mock_make_commit.assert_not_called()
  763. mock_send_pull_request.assert_not_called()
  764. @patch('openhands.resolver.send_pull_request.load_all_resolver_outputs')
  765. @patch('openhands.resolver.send_pull_request.process_single_issue')
  766. def test_process_all_successful_issues(
  767. mock_process_single_issue, mock_load_all_resolver_outputs, mock_llm_config
  768. ):
  769. # Create ResolverOutput objects with properly initialized GithubIssue instances
  770. resolver_output_1 = ResolverOutput(
  771. issue=GithubIssue(
  772. owner='test-owner',
  773. repo='test-repo',
  774. number=1,
  775. title='Issue 1',
  776. body='Body 1',
  777. ),
  778. issue_type='issue',
  779. instruction='Test instruction 1',
  780. base_commit='def456',
  781. git_patch='Test patch 1',
  782. history=[],
  783. metrics={},
  784. success=True,
  785. comment_success=None,
  786. success_explanation='Test success 1',
  787. error=None,
  788. )
  789. resolver_output_2 = ResolverOutput(
  790. issue=GithubIssue(
  791. owner='test-owner',
  792. repo='test-repo',
  793. number=2,
  794. title='Issue 2',
  795. body='Body 2',
  796. ),
  797. issue_type='issue',
  798. instruction='Test instruction 2',
  799. base_commit='ghi789',
  800. git_patch='Test patch 2',
  801. history=[],
  802. metrics={},
  803. success=False,
  804. comment_success=None,
  805. success_explanation='',
  806. error='Test error 2',
  807. )
  808. resolver_output_3 = ResolverOutput(
  809. issue=GithubIssue(
  810. owner='test-owner',
  811. repo='test-repo',
  812. number=3,
  813. title='Issue 3',
  814. body='Body 3',
  815. ),
  816. issue_type='issue',
  817. instruction='Test instruction 3',
  818. base_commit='jkl012',
  819. git_patch='Test patch 3',
  820. history=[],
  821. metrics={},
  822. success=True,
  823. comment_success=None,
  824. success_explanation='Test success 3',
  825. error=None,
  826. )
  827. mock_load_all_resolver_outputs.return_value = [
  828. resolver_output_1,
  829. resolver_output_2,
  830. resolver_output_3,
  831. ]
  832. # Call the function
  833. process_all_successful_issues(
  834. 'output_dir',
  835. 'github_token',
  836. 'github_username',
  837. 'draft',
  838. mock_llm_config, # llm_config
  839. None, # fork_owner
  840. )
  841. # Assert that process_single_issue was called for successful issues only
  842. assert mock_process_single_issue.call_count == 2
  843. # Check that the function was called with the correct arguments for successful issues
  844. mock_process_single_issue.assert_has_calls(
  845. [
  846. call(
  847. 'output_dir',
  848. resolver_output_1,
  849. 'github_token',
  850. 'github_username',
  851. 'draft',
  852. mock_llm_config,
  853. None,
  854. False,
  855. None,
  856. ),
  857. call(
  858. 'output_dir',
  859. resolver_output_3,
  860. 'github_token',
  861. 'github_username',
  862. 'draft',
  863. mock_llm_config,
  864. None,
  865. False,
  866. None,
  867. ),
  868. ]
  869. )
  870. # Add more assertions as needed to verify the behavior of the function
  871. @patch('requests.get')
  872. @patch('subprocess.run')
  873. def test_send_pull_request_branch_naming(
  874. mock_run, mock_get, mock_github_issue, mock_output_dir
  875. ):
  876. repo_path = os.path.join(mock_output_dir, 'repo')
  877. # Mock API responses
  878. mock_get.side_effect = [
  879. MagicMock(status_code=200), # First branch exists
  880. MagicMock(status_code=200), # Second branch exists
  881. MagicMock(status_code=404), # Third branch doesn't exist
  882. MagicMock(json=lambda: {'default_branch': 'main'}), # Get default branch
  883. ]
  884. # Mock subprocess.run calls
  885. mock_run.side_effect = [
  886. MagicMock(returncode=0), # git checkout -b
  887. MagicMock(returncode=0), # git push
  888. ]
  889. # Call the function
  890. result = send_pull_request(
  891. github_issue=mock_github_issue,
  892. github_token='test-token',
  893. github_username='test-user',
  894. patch_dir=repo_path,
  895. pr_type='branch',
  896. )
  897. # Assert API calls
  898. assert mock_get.call_count == 4
  899. # Check branch creation and push
  900. assert mock_run.call_count == 2
  901. checkout_call, push_call = mock_run.call_args_list
  902. assert checkout_call == call(
  903. ['git', '-C', repo_path, 'checkout', '-b', 'openhands-fix-issue-42-try3'],
  904. capture_output=True,
  905. text=True,
  906. )
  907. assert push_call == call(
  908. [
  909. 'git',
  910. '-C',
  911. repo_path,
  912. 'push',
  913. 'https://test-user:test-token@github.com/test-owner/test-repo.git',
  914. 'openhands-fix-issue-42-try3',
  915. ],
  916. capture_output=True,
  917. text=True,
  918. )
  919. # Check the result
  920. assert (
  921. result
  922. == 'https://github.com/test-owner/test-repo/compare/openhands-fix-issue-42-try3?expand=1'
  923. )
  924. @patch('openhands.resolver.send_pull_request.argparse.ArgumentParser')
  925. @patch('openhands.resolver.send_pull_request.process_all_successful_issues')
  926. @patch('openhands.resolver.send_pull_request.process_single_issue')
  927. @patch('openhands.resolver.send_pull_request.load_single_resolver_output')
  928. @patch('os.path.exists')
  929. @patch('os.getenv')
  930. def test_main(
  931. mock_getenv,
  932. mock_path_exists,
  933. mock_load_single_resolver_output,
  934. mock_process_single_issue,
  935. mock_process_all_successful_issues,
  936. mock_parser,
  937. ):
  938. from openhands.resolver.send_pull_request import main
  939. # Setup mock parser
  940. mock_args = MagicMock()
  941. mock_args.github_token = None
  942. mock_args.github_username = 'mock_username'
  943. mock_args.output_dir = '/mock/output'
  944. mock_args.pr_type = 'draft'
  945. mock_args.issue_number = '42'
  946. mock_args.fork_owner = None
  947. mock_args.send_on_failure = False
  948. mock_args.llm_model = 'mock_model'
  949. mock_args.llm_base_url = 'mock_url'
  950. mock_args.llm_api_key = 'mock_key'
  951. mock_args.target_branch = None
  952. mock_args.reviewer = None
  953. mock_parser.return_value.parse_args.return_value = mock_args
  954. # Setup environment variables
  955. mock_getenv.side_effect = (
  956. lambda key, default=None: 'mock_token' if key == 'GITHUB_TOKEN' else default
  957. )
  958. # Setup path exists
  959. mock_path_exists.return_value = True
  960. # Setup mock resolver output
  961. mock_resolver_output = MagicMock()
  962. mock_load_single_resolver_output.return_value = mock_resolver_output
  963. # Run main function
  964. main()
  965. llm_config = LLMConfig(
  966. model=mock_args.llm_model,
  967. base_url=mock_args.llm_base_url,
  968. api_key=mock_args.llm_api_key,
  969. )
  970. # Use any_call instead of assert_called_with for more flexible matching
  971. assert mock_process_single_issue.call_args == call(
  972. '/mock/output',
  973. mock_resolver_output,
  974. 'mock_token',
  975. 'mock_username',
  976. 'draft',
  977. llm_config,
  978. None,
  979. False,
  980. mock_args.target_branch,
  981. mock_args.reviewer,
  982. )
  983. # Other assertions
  984. mock_parser.assert_called_once()
  985. mock_getenv.assert_any_call('GITHUB_TOKEN')
  986. mock_path_exists.assert_called_with('/mock/output')
  987. mock_load_single_resolver_output.assert_called_with('/mock/output/output.jsonl', 42)
  988. # Test for 'all_successful' issue number
  989. mock_args.issue_number = 'all_successful'
  990. main()
  991. mock_process_all_successful_issues.assert_called_with(
  992. '/mock/output',
  993. 'mock_token',
  994. 'mock_username',
  995. 'draft',
  996. llm_config,
  997. None,
  998. )
  999. # Test for invalid issue number
  1000. mock_args.issue_number = 'invalid'
  1001. with pytest.raises(ValueError):
  1002. main()
  1003. @patch('subprocess.run')
  1004. def test_make_commit_escapes_issue_title(mock_subprocess_run):
  1005. # Setup
  1006. repo_dir = '/path/to/repo'
  1007. issue = GithubIssue(
  1008. owner='test-owner',
  1009. repo='test-repo',
  1010. number=42,
  1011. title='Issue with "quotes" and $pecial characters',
  1012. body='Test body',
  1013. )
  1014. # Mock subprocess.run to return success for all calls
  1015. mock_subprocess_run.return_value = MagicMock(
  1016. returncode=0, stdout='sample output', stderr=''
  1017. )
  1018. # Call the function
  1019. issue_type = 'issue'
  1020. make_commit(repo_dir, issue, issue_type)
  1021. # Assert that subprocess.run was called with the correct arguments
  1022. calls = mock_subprocess_run.call_args_list
  1023. assert len(calls) == 4 # git config check, git add, git commit
  1024. # Check the git commit call
  1025. git_commit_call = calls[3][0][0]
  1026. expected_commit_message = (
  1027. 'Fix issue #42: Issue with "quotes" and $pecial characters'
  1028. )
  1029. assert [
  1030. 'git',
  1031. '-C',
  1032. '/path/to/repo',
  1033. 'commit',
  1034. '-m',
  1035. expected_commit_message,
  1036. ] == git_commit_call
  1037. @patch('subprocess.run')
  1038. def test_make_commit_no_changes(mock_subprocess_run):
  1039. # Setup
  1040. repo_dir = '/path/to/repo'
  1041. issue = GithubIssue(
  1042. owner='test-owner',
  1043. repo='test-repo',
  1044. number=42,
  1045. title='Issue with no changes',
  1046. body='Test body',
  1047. )
  1048. # Mock subprocess.run to simulate no changes in the repo
  1049. mock_subprocess_run.side_effect = [
  1050. MagicMock(returncode=0),
  1051. MagicMock(returncode=0),
  1052. MagicMock(returncode=1, stdout=''), # git status --porcelain (no changes)
  1053. ]
  1054. with pytest.raises(
  1055. RuntimeError, match='ERROR: Openhands failed to make code changes.'
  1056. ):
  1057. make_commit(repo_dir, issue, 'issue')
  1058. # Check that subprocess.run was called for checking git status and add, but not commit
  1059. assert mock_subprocess_run.call_count == 3
  1060. git_status_call = mock_subprocess_run.call_args_list[2][0][0]
  1061. assert f'git -C {repo_dir} status --porcelain' in git_status_call
  1062. def test_apply_patch_rename_directory(mock_output_dir):
  1063. # Create a sample directory structure
  1064. old_dir = os.path.join(mock_output_dir, 'prompts', 'resolve')
  1065. os.makedirs(old_dir)
  1066. # Create test files
  1067. test_files = [
  1068. 'issue-success-check.jinja',
  1069. 'pr-feedback-check.jinja',
  1070. 'pr-thread-check.jinja',
  1071. ]
  1072. for filename in test_files:
  1073. file_path = os.path.join(old_dir, filename)
  1074. with open(file_path, 'w') as f:
  1075. f.write(f'Content of {filename}')
  1076. # Create a patch that renames the directory
  1077. patch_content = """diff --git a/prompts/resolve/issue-success-check.jinja b/prompts/guess_success/issue-success-check.jinja
  1078. similarity index 100%
  1079. rename from prompts/resolve/issue-success-check.jinja
  1080. rename to prompts/guess_success/issue-success-check.jinja
  1081. diff --git a/prompts/resolve/pr-feedback-check.jinja b/prompts/guess_success/pr-feedback-check.jinja
  1082. similarity index 100%
  1083. rename from prompts/resolve/pr-feedback-check.jinja
  1084. rename to prompts/guess_success/pr-feedback-check.jinja
  1085. diff --git a/prompts/resolve/pr-thread-check.jinja b/prompts/guess_success/pr-thread-check.jinja
  1086. similarity index 100%
  1087. rename from prompts/resolve/pr-thread-check.jinja
  1088. rename to prompts/guess_success/pr-thread-check.jinja"""
  1089. # Apply the patch
  1090. apply_patch(mock_output_dir, patch_content)
  1091. # Check if files were moved correctly
  1092. new_dir = os.path.join(mock_output_dir, 'prompts', 'guess_success')
  1093. assert not os.path.exists(old_dir), 'Old directory still exists'
  1094. assert os.path.exists(new_dir), 'New directory was not created'
  1095. # Check if all files were moved and content preserved
  1096. for filename in test_files:
  1097. old_path = os.path.join(old_dir, filename)
  1098. new_path = os.path.join(new_dir, filename)
  1099. assert not os.path.exists(old_path), f'Old file {filename} still exists'
  1100. assert os.path.exists(new_path), f'New file {filename} was not created'
  1101. with open(new_path, 'r') as f:
  1102. content = f.read()
  1103. assert content == f'Content of {filename}', f'Content mismatch for {filename}'