test_send_pull_request.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284
  1. import os
  2. import tempfile
  3. from unittest.mock import MagicMock, call, patch
  4. import pytest
  5. from openhands.core.config import LLMConfig
  6. from openhands.resolver.github_issue import ReviewThread
  7. from openhands.resolver.resolver_output import GithubIssue, ResolverOutput
  8. from openhands.resolver.send_pull_request import (
  9. apply_patch,
  10. initialize_repo,
  11. load_single_resolver_output,
  12. make_commit,
  13. process_all_successful_issues,
  14. process_single_issue,
  15. reply_to_comment,
  16. send_pull_request,
  17. update_existing_pull_request,
  18. )
  19. @pytest.fixture
  20. def mock_output_dir():
  21. with tempfile.TemporaryDirectory() as temp_dir:
  22. repo_path = os.path.join(temp_dir, 'repo')
  23. # Initialize a GitHub repo in "repo" and add a commit with "README.md"
  24. os.makedirs(repo_path)
  25. os.system(f'git init {repo_path}')
  26. readme_path = os.path.join(repo_path, 'README.md')
  27. with open(readme_path, 'w') as f:
  28. f.write('hello world')
  29. os.system(f'git -C {repo_path} add README.md')
  30. os.system(f"git -C {repo_path} commit -m 'Initial commit'")
  31. yield temp_dir
  32. @pytest.fixture
  33. def mock_github_issue():
  34. return GithubIssue(
  35. number=42,
  36. title='Test Issue',
  37. owner='test-owner',
  38. repo='test-repo',
  39. body='Test body',
  40. )
  41. @pytest.fixture
  42. def mock_llm_config():
  43. return LLMConfig()
  44. def test_load_single_resolver_output():
  45. mock_output_jsonl = 'tests/unit/resolver/mock_output/output.jsonl'
  46. # Test loading an existing issue
  47. resolver_output = load_single_resolver_output(mock_output_jsonl, 5)
  48. assert isinstance(resolver_output, ResolverOutput)
  49. assert resolver_output.issue.number == 5
  50. assert resolver_output.issue.title == 'Add MIT license'
  51. assert resolver_output.issue.owner == 'neubig'
  52. assert resolver_output.issue.repo == 'pr-viewer'
  53. # Test loading a non-existent issue
  54. with pytest.raises(ValueError):
  55. load_single_resolver_output(mock_output_jsonl, 999)
  56. def test_apply_patch(mock_output_dir):
  57. # Create a sample file in the mock repo
  58. sample_file = os.path.join(mock_output_dir, 'sample.txt')
  59. with open(sample_file, 'w') as f:
  60. f.write('Original content')
  61. # Create a sample patch
  62. patch_content = """
  63. diff --git a/sample.txt b/sample.txt
  64. index 9daeafb..b02def2 100644
  65. --- a/sample.txt
  66. +++ b/sample.txt
  67. @@ -1 +1,2 @@
  68. -Original content
  69. +Updated content
  70. +New line
  71. """
  72. # Apply the patch
  73. apply_patch(mock_output_dir, patch_content)
  74. # Check if the file was updated correctly
  75. with open(sample_file, 'r') as f:
  76. updated_content = f.read()
  77. assert updated_content.strip() == 'Updated content\nNew line'.strip()
  78. def test_apply_patch_preserves_line_endings(mock_output_dir):
  79. # Create sample files with different line endings
  80. unix_file = os.path.join(mock_output_dir, 'unix_style.txt')
  81. dos_file = os.path.join(mock_output_dir, 'dos_style.txt')
  82. with open(unix_file, 'w', newline='\n') as f:
  83. f.write('Line 1\nLine 2\nLine 3')
  84. with open(dos_file, 'w', newline='\r\n') as f:
  85. f.write('Line 1\r\nLine 2\r\nLine 3')
  86. # Create patches for both files
  87. unix_patch = """
  88. diff --git a/unix_style.txt b/unix_style.txt
  89. index 9daeafb..b02def2 100644
  90. --- a/unix_style.txt
  91. +++ b/unix_style.txt
  92. @@ -1,3 +1,3 @@
  93. Line 1
  94. -Line 2
  95. +Updated Line 2
  96. Line 3
  97. """
  98. dos_patch = """
  99. diff --git a/dos_style.txt b/dos_style.txt
  100. index 9daeafb..b02def2 100644
  101. --- a/dos_style.txt
  102. +++ b/dos_style.txt
  103. @@ -1,3 +1,3 @@
  104. Line 1
  105. -Line 2
  106. +Updated Line 2
  107. Line 3
  108. """
  109. # Apply patches
  110. apply_patch(mock_output_dir, unix_patch)
  111. apply_patch(mock_output_dir, dos_patch)
  112. # Check if line endings are preserved
  113. with open(unix_file, 'rb') as f:
  114. unix_content = f.read()
  115. with open(dos_file, 'rb') as f:
  116. dos_content = f.read()
  117. assert (
  118. b'\r\n' not in unix_content
  119. ), 'Unix-style line endings were changed to DOS-style'
  120. assert b'\r\n' in dos_content, 'DOS-style line endings were changed to Unix-style'
  121. # Check if content was updated correctly
  122. assert unix_content.decode('utf-8').split('\n')[1] == 'Updated Line 2'
  123. assert dos_content.decode('utf-8').split('\r\n')[1] == 'Updated Line 2'
  124. def test_apply_patch_create_new_file(mock_output_dir):
  125. # Create a patch that adds a new file
  126. patch_content = """
  127. diff --git a/new_file.txt b/new_file.txt
  128. new file mode 100644
  129. index 0000000..3b18e51
  130. --- /dev/null
  131. +++ b/new_file.txt
  132. @@ -0,0 +1 @@
  133. +hello world
  134. """
  135. # Apply the patch
  136. apply_patch(mock_output_dir, patch_content)
  137. # Check if the new file was created
  138. new_file_path = os.path.join(mock_output_dir, 'new_file.txt')
  139. assert os.path.exists(new_file_path), 'New file was not created'
  140. # Check if the file content is correct
  141. with open(new_file_path, 'r') as f:
  142. content = f.read().strip()
  143. assert content == 'hello world', 'File content is incorrect'
  144. def test_apply_patch_rename_file(mock_output_dir):
  145. # Create a sample file in the mock repo
  146. old_file = os.path.join(mock_output_dir, 'old_name.txt')
  147. with open(old_file, 'w') as f:
  148. f.write('This file will be renamed')
  149. # Create a patch that renames the file
  150. patch_content = """diff --git a/old_name.txt b/new_name.txt
  151. similarity index 100%
  152. rename from old_name.txt
  153. rename to new_name.txt"""
  154. # Apply the patch
  155. apply_patch(mock_output_dir, patch_content)
  156. # Check if the file was renamed
  157. new_file = os.path.join(mock_output_dir, 'new_name.txt')
  158. assert not os.path.exists(old_file), 'Old file still exists'
  159. assert os.path.exists(new_file), 'New file was not created'
  160. # Check if the content is preserved
  161. with open(new_file, 'r') as f:
  162. content = f.read()
  163. assert content == 'This file will be renamed'
  164. def test_apply_patch_delete_file(mock_output_dir):
  165. # Create a sample file in the mock repo
  166. sample_file = os.path.join(mock_output_dir, 'to_be_deleted.txt')
  167. with open(sample_file, 'w') as f:
  168. f.write('This file will be deleted')
  169. # Create a patch that deletes the file
  170. patch_content = """
  171. diff --git a/to_be_deleted.txt b/to_be_deleted.txt
  172. deleted file mode 100644
  173. index 9daeafb..0000000
  174. --- a/to_be_deleted.txt
  175. +++ /dev/null
  176. @@ -1 +0,0 @@
  177. -This file will be deleted
  178. """
  179. # Apply the patch
  180. apply_patch(mock_output_dir, patch_content)
  181. # Check if the file was deleted
  182. assert not os.path.exists(sample_file), 'File was not deleted'
  183. def test_initialize_repo(mock_output_dir):
  184. issue_type = 'issue'
  185. # Copy the repo to patches
  186. ISSUE_NUMBER = 3
  187. initialize_repo(mock_output_dir, ISSUE_NUMBER, issue_type)
  188. patches_dir = os.path.join(mock_output_dir, 'patches', f'issue_{ISSUE_NUMBER}')
  189. # Check if files were copied correctly
  190. assert os.path.exists(os.path.join(patches_dir, 'README.md'))
  191. # Check file contents
  192. with open(os.path.join(patches_dir, 'README.md'), 'r') as f:
  193. assert f.read() == 'hello world'
  194. @patch('openhands.resolver.send_pull_request.reply_to_comment')
  195. @patch('requests.post')
  196. @patch('subprocess.run')
  197. @patch('openhands.resolver.send_pull_request.LLM')
  198. def test_update_existing_pull_request(
  199. mock_llm_class,
  200. mock_subprocess_run,
  201. mock_requests_post,
  202. mock_reply_to_comment,
  203. ):
  204. # Arrange: Set up test data
  205. github_issue = GithubIssue(
  206. owner='test-owner',
  207. repo='test-repo',
  208. number=1,
  209. title='Test PR',
  210. body='This is a test PR',
  211. thread_ids=['comment1', 'comment2'],
  212. head_branch='test-branch',
  213. )
  214. github_token = 'test-token'
  215. github_username = 'test-user'
  216. patch_dir = '/path/to/patch'
  217. additional_message = '["Fixed bug in function A", "Updated documentation for B"]'
  218. # Mock the subprocess.run call for git push
  219. mock_subprocess_run.return_value = MagicMock(returncode=0)
  220. # Mock the requests.post call for adding a PR comment
  221. mock_requests_post.return_value.status_code = 201
  222. # Mock LLM instance and completion call
  223. mock_llm_instance = MagicMock()
  224. mock_completion_response = MagicMock()
  225. mock_completion_response.choices = [
  226. MagicMock(message=MagicMock(content='This is an issue resolution.'))
  227. ]
  228. mock_llm_instance.completion.return_value = mock_completion_response
  229. mock_llm_class.return_value = mock_llm_instance
  230. llm_config = LLMConfig()
  231. # Act: Call the function without comment_message to test auto-generation
  232. result = update_existing_pull_request(
  233. github_issue,
  234. github_token,
  235. github_username,
  236. patch_dir,
  237. llm_config,
  238. comment_message=None,
  239. additional_message=additional_message,
  240. )
  241. # Assert: Check if the git push command was executed
  242. push_command = (
  243. f'git -C {patch_dir} push '
  244. f'https://{github_username}:{github_token}@github.com/'
  245. f'{github_issue.owner}/{github_issue.repo}.git {github_issue.head_branch}'
  246. )
  247. mock_subprocess_run.assert_called_once_with(
  248. push_command, shell=True, capture_output=True, text=True
  249. )
  250. # Assert: Check if the auto-generated comment was posted to the PR
  251. comment_url = f'https://api.github.com/repos/{github_issue.owner}/{github_issue.repo}/issues/{github_issue.number}/comments'
  252. expected_comment = 'This is an issue resolution.'
  253. mock_requests_post.assert_called_once_with(
  254. comment_url,
  255. headers={
  256. 'Authorization': f'token {github_token}',
  257. 'Accept': 'application/vnd.github.v3+json',
  258. },
  259. json={'body': expected_comment},
  260. )
  261. # Assert: Check if the reply_to_comment function was called for each thread ID
  262. mock_reply_to_comment.assert_has_calls(
  263. [
  264. call(github_token, 'comment1', 'Fixed bug in function A'),
  265. call(github_token, 'comment2', 'Updated documentation for B'),
  266. ]
  267. )
  268. # Assert: Check the returned PR URL
  269. assert (
  270. result
  271. == f'https://github.com/{github_issue.owner}/{github_issue.repo}/pull/{github_issue.number}'
  272. )
  273. @pytest.mark.parametrize(
  274. 'pr_type,target_branch,pr_title',
  275. [
  276. ('branch', None, None),
  277. ('draft', None, None),
  278. ('ready', None, None),
  279. ('branch', 'feature', None),
  280. ('draft', 'develop', None),
  281. ('ready', 'staging', None),
  282. ('ready', None, 'Custom PR Title'),
  283. ('draft', 'develop', 'Another Custom Title'),
  284. ],
  285. )
  286. @patch('subprocess.run')
  287. @patch('requests.post')
  288. @patch('requests.get')
  289. def test_send_pull_request(
  290. mock_get,
  291. mock_post,
  292. mock_run,
  293. mock_github_issue,
  294. mock_output_dir,
  295. pr_type,
  296. target_branch,
  297. pr_title,
  298. ):
  299. repo_path = os.path.join(mock_output_dir, 'repo')
  300. # Mock API responses based on whether target_branch is specified
  301. if target_branch:
  302. mock_get.side_effect = [
  303. MagicMock(status_code=404), # Branch doesn't exist
  304. MagicMock(status_code=200), # Target branch exists
  305. ]
  306. else:
  307. mock_get.side_effect = [
  308. MagicMock(status_code=404), # Branch doesn't exist
  309. MagicMock(json=lambda: {'default_branch': 'main'}), # Get default branch
  310. ]
  311. mock_post.return_value.json.return_value = {
  312. 'html_url': 'https://github.com/test-owner/test-repo/pull/1'
  313. }
  314. # Mock subprocess.run calls
  315. mock_run.side_effect = [
  316. MagicMock(returncode=0), # git checkout -b
  317. MagicMock(returncode=0), # git push
  318. ]
  319. # Call the function
  320. result = send_pull_request(
  321. github_issue=mock_github_issue,
  322. github_token='test-token',
  323. github_username='test-user',
  324. patch_dir=repo_path,
  325. pr_type=pr_type,
  326. target_branch=target_branch,
  327. pr_title=pr_title,
  328. )
  329. # Assert API calls
  330. expected_get_calls = 2
  331. assert mock_get.call_count == expected_get_calls
  332. # Check branch creation and push
  333. assert mock_run.call_count == 2
  334. checkout_call, push_call = mock_run.call_args_list
  335. assert checkout_call == call(
  336. ['git', '-C', repo_path, 'checkout', '-b', 'openhands-fix-issue-42'],
  337. capture_output=True,
  338. text=True,
  339. )
  340. assert push_call == call(
  341. [
  342. 'git',
  343. '-C',
  344. repo_path,
  345. 'push',
  346. 'https://test-user:test-token@github.com/test-owner/test-repo.git',
  347. 'openhands-fix-issue-42',
  348. ],
  349. capture_output=True,
  350. text=True,
  351. )
  352. # Check PR creation based on pr_type
  353. if pr_type == 'branch':
  354. assert (
  355. result
  356. == 'https://github.com/test-owner/test-repo/compare/openhands-fix-issue-42?expand=1'
  357. )
  358. mock_post.assert_not_called()
  359. else:
  360. assert result == 'https://github.com/test-owner/test-repo/pull/1'
  361. mock_post.assert_called_once()
  362. post_data = mock_post.call_args[1]['json']
  363. expected_title = pr_title if pr_title else 'Fix issue #42: Test Issue'
  364. assert post_data['title'] == expected_title
  365. assert post_data['body'].startswith('This pull request fixes #42.')
  366. assert post_data['head'] == 'openhands-fix-issue-42'
  367. assert post_data['base'] == (target_branch if target_branch else 'main')
  368. assert post_data['draft'] == (pr_type == 'draft')
  369. @patch('subprocess.run')
  370. @patch('requests.post')
  371. @patch('requests.get')
  372. def test_send_pull_request_with_reviewer(
  373. mock_get, mock_post, mock_run, mock_github_issue, mock_output_dir
  374. ):
  375. repo_path = os.path.join(mock_output_dir, 'repo')
  376. reviewer = 'test-reviewer'
  377. # Mock API responses
  378. mock_get.side_effect = [
  379. MagicMock(status_code=404), # Branch doesn't exist
  380. MagicMock(json=lambda: {'default_branch': 'main'}), # Get default branch
  381. ]
  382. # Mock PR creation response
  383. mock_post.side_effect = [
  384. MagicMock(
  385. status_code=201,
  386. json=lambda: {
  387. 'html_url': 'https://github.com/test-owner/test-repo/pull/1',
  388. 'number': 1,
  389. },
  390. ), # PR creation
  391. MagicMock(status_code=201), # Reviewer request
  392. ]
  393. # Mock subprocess.run calls
  394. mock_run.side_effect = [
  395. MagicMock(returncode=0), # git checkout -b
  396. MagicMock(returncode=0), # git push
  397. ]
  398. # Call the function with reviewer
  399. result = send_pull_request(
  400. github_issue=mock_github_issue,
  401. github_token='test-token',
  402. github_username='test-user',
  403. patch_dir=repo_path,
  404. pr_type='ready',
  405. reviewer=reviewer,
  406. )
  407. # Assert API calls
  408. assert mock_get.call_count == 2
  409. assert mock_post.call_count == 2
  410. # Check PR creation
  411. pr_create_call = mock_post.call_args_list[0]
  412. assert pr_create_call[1]['json']['title'] == 'Fix issue #42: Test Issue'
  413. # Check reviewer request
  414. reviewer_request_call = mock_post.call_args_list[1]
  415. assert (
  416. reviewer_request_call[0][0]
  417. == 'https://api.github.com/repos/test-owner/test-repo/pulls/1/requested_reviewers'
  418. )
  419. assert reviewer_request_call[1]['json'] == {'reviewers': ['test-reviewer']}
  420. # Check the result URL
  421. assert result == 'https://github.com/test-owner/test-repo/pull/1'
  422. @patch('requests.get')
  423. def test_send_pull_request_invalid_target_branch(
  424. mock_get, mock_github_issue, mock_output_dir
  425. ):
  426. """Test that an error is raised when specifying a non-existent target branch"""
  427. repo_path = os.path.join(mock_output_dir, 'repo')
  428. # Mock API response for non-existent branch
  429. mock_get.side_effect = [
  430. MagicMock(status_code=404), # Branch doesn't exist
  431. MagicMock(status_code=404), # Target branch doesn't exist
  432. ]
  433. # Test that ValueError is raised when target branch doesn't exist
  434. with pytest.raises(
  435. ValueError, match='Target branch nonexistent-branch does not exist'
  436. ):
  437. send_pull_request(
  438. github_issue=mock_github_issue,
  439. github_token='test-token',
  440. github_username='test-user',
  441. patch_dir=repo_path,
  442. pr_type='ready',
  443. target_branch='nonexistent-branch',
  444. )
  445. # Verify API calls
  446. assert mock_get.call_count == 2
  447. @patch('subprocess.run')
  448. @patch('requests.post')
  449. @patch('requests.get')
  450. def test_send_pull_request_git_push_failure(
  451. mock_get, mock_post, mock_run, mock_github_issue, mock_output_dir
  452. ):
  453. repo_path = os.path.join(mock_output_dir, 'repo')
  454. # Mock API responses
  455. mock_get.return_value = MagicMock(json=lambda: {'default_branch': 'main'})
  456. # Mock the subprocess.run calls
  457. mock_run.side_effect = [
  458. MagicMock(returncode=0), # git checkout -b
  459. MagicMock(returncode=1, stderr='Error: failed to push some refs'), # git push
  460. ]
  461. # Test that RuntimeError is raised when git push fails
  462. with pytest.raises(
  463. RuntimeError, match='Failed to push changes to the remote repository'
  464. ):
  465. send_pull_request(
  466. github_issue=mock_github_issue,
  467. github_token='test-token',
  468. github_username='test-user',
  469. patch_dir=repo_path,
  470. pr_type='ready',
  471. )
  472. # Assert that subprocess.run was called twice
  473. assert mock_run.call_count == 2
  474. # Check the git checkout -b command
  475. checkout_call = mock_run.call_args_list[0]
  476. assert checkout_call[0][0] == [
  477. 'git',
  478. '-C',
  479. repo_path,
  480. 'checkout',
  481. '-b',
  482. 'openhands-fix-issue-42',
  483. ]
  484. # Check the git push command
  485. push_call = mock_run.call_args_list[1]
  486. assert push_call[0][0] == [
  487. 'git',
  488. '-C',
  489. repo_path,
  490. 'push',
  491. 'https://test-user:test-token@github.com/test-owner/test-repo.git',
  492. 'openhands-fix-issue-42',
  493. ]
  494. # Assert that no pull request was created
  495. mock_post.assert_not_called()
  496. @patch('subprocess.run')
  497. @patch('requests.post')
  498. @patch('requests.get')
  499. def test_send_pull_request_permission_error(
  500. mock_get, mock_post, mock_run, mock_github_issue, mock_output_dir
  501. ):
  502. repo_path = os.path.join(mock_output_dir, 'repo')
  503. # Mock API responses
  504. mock_get.return_value = MagicMock(json=lambda: {'default_branch': 'main'})
  505. mock_post.return_value.status_code = 403
  506. # Mock subprocess.run calls
  507. mock_run.side_effect = [
  508. MagicMock(returncode=0), # git checkout -b
  509. MagicMock(returncode=0), # git push
  510. ]
  511. # Test that RuntimeError is raised when PR creation fails due to permissions
  512. with pytest.raises(
  513. RuntimeError, match='Failed to create pull request due to missing permissions.'
  514. ):
  515. send_pull_request(
  516. github_issue=mock_github_issue,
  517. github_token='test-token',
  518. github_username='test-user',
  519. patch_dir=repo_path,
  520. pr_type='ready',
  521. )
  522. # Assert that the branch was created and pushed
  523. assert mock_run.call_count == 2
  524. mock_post.assert_called_once()
  525. @patch('requests.post')
  526. def test_reply_to_comment(mock_post):
  527. # Arrange: set up the test data
  528. github_token = 'test_token'
  529. comment_id = 'test_comment_id'
  530. reply = 'This is a test reply.'
  531. # Mock the response from the GraphQL API
  532. mock_response = MagicMock()
  533. mock_response.status_code = 200
  534. mock_response.json.return_value = {
  535. 'data': {
  536. 'addPullRequestReviewThreadReply': {
  537. 'comment': {
  538. 'id': 'test_reply_id',
  539. 'body': 'Openhands fix success summary\n\n\nThis is a test reply.',
  540. 'createdAt': '2024-10-01T12:34:56Z',
  541. }
  542. }
  543. }
  544. }
  545. mock_post.return_value = mock_response
  546. # Act: call the function
  547. reply_to_comment(github_token, comment_id, reply)
  548. # Assert: check that the POST request was made with the correct parameters
  549. query = """
  550. mutation($body: String!, $pullRequestReviewThreadId: ID!) {
  551. addPullRequestReviewThreadReply(input: { body: $body, pullRequestReviewThreadId: $pullRequestReviewThreadId }) {
  552. comment {
  553. id
  554. body
  555. createdAt
  556. }
  557. }
  558. }
  559. """
  560. expected_variables = {
  561. 'body': 'Openhands fix success summary\n\n\nThis is a test reply.',
  562. 'pullRequestReviewThreadId': comment_id,
  563. }
  564. # Check that the correct request was made to the API
  565. mock_post.assert_called_once_with(
  566. 'https://api.github.com/graphql',
  567. json={'query': query, 'variables': expected_variables},
  568. headers={
  569. 'Authorization': f'Bearer {github_token}',
  570. 'Content-Type': 'application/json',
  571. },
  572. )
  573. # Check that the response status was checked (via response.raise_for_status)
  574. mock_response.raise_for_status.assert_called_once()
  575. @patch('openhands.resolver.send_pull_request.initialize_repo')
  576. @patch('openhands.resolver.send_pull_request.apply_patch')
  577. @patch('openhands.resolver.send_pull_request.update_existing_pull_request')
  578. @patch('openhands.resolver.send_pull_request.make_commit')
  579. def test_process_single_pr_update(
  580. mock_make_commit,
  581. mock_update_existing_pull_request,
  582. mock_apply_patch,
  583. mock_initialize_repo,
  584. mock_output_dir,
  585. mock_llm_config,
  586. ):
  587. # Initialize test data
  588. github_token = 'test_token'
  589. github_username = 'test_user'
  590. pr_type = 'draft'
  591. resolver_output = ResolverOutput(
  592. issue=GithubIssue(
  593. owner='test-owner',
  594. repo='test-repo',
  595. number=1,
  596. title='Issue 1',
  597. body='Body 1',
  598. closing_issues=[],
  599. review_threads=[
  600. ReviewThread(comment='review comment for feedback', files=[])
  601. ],
  602. thread_ids=['1'],
  603. head_branch='branch 1',
  604. ),
  605. issue_type='pr',
  606. instruction='Test instruction 1',
  607. base_commit='def456',
  608. git_patch='Test patch 1',
  609. history=[],
  610. metrics={},
  611. success=True,
  612. comment_success=None,
  613. success_explanation='[Test success 1]',
  614. error=None,
  615. )
  616. mock_update_existing_pull_request.return_value = (
  617. 'https://github.com/test-owner/test-repo/pull/1'
  618. )
  619. mock_initialize_repo.return_value = f'{mock_output_dir}/patches/pr_1'
  620. process_single_issue(
  621. mock_output_dir,
  622. resolver_output,
  623. github_token,
  624. github_username,
  625. pr_type,
  626. mock_llm_config,
  627. None,
  628. False,
  629. None,
  630. )
  631. mock_initialize_repo.assert_called_once_with(mock_output_dir, 1, 'pr', 'branch 1')
  632. mock_apply_patch.assert_called_once_with(
  633. f'{mock_output_dir}/patches/pr_1', resolver_output.git_patch
  634. )
  635. mock_make_commit.assert_called_once_with(
  636. f'{mock_output_dir}/patches/pr_1', resolver_output.issue, 'pr'
  637. )
  638. mock_update_existing_pull_request.assert_called_once_with(
  639. github_issue=resolver_output.issue,
  640. github_token=github_token,
  641. github_username=github_username,
  642. patch_dir=f'{mock_output_dir}/patches/pr_1',
  643. additional_message='[Test success 1]',
  644. llm_config=mock_llm_config,
  645. )
  646. @patch('openhands.resolver.send_pull_request.initialize_repo')
  647. @patch('openhands.resolver.send_pull_request.apply_patch')
  648. @patch('openhands.resolver.send_pull_request.send_pull_request')
  649. @patch('openhands.resolver.send_pull_request.make_commit')
  650. def test_process_single_issue(
  651. mock_make_commit,
  652. mock_send_pull_request,
  653. mock_apply_patch,
  654. mock_initialize_repo,
  655. mock_output_dir,
  656. mock_llm_config,
  657. ):
  658. # Initialize test data
  659. github_token = 'test_token'
  660. github_username = 'test_user'
  661. pr_type = 'draft'
  662. resolver_output = ResolverOutput(
  663. issue=GithubIssue(
  664. owner='test-owner',
  665. repo='test-repo',
  666. number=1,
  667. title='Issue 1',
  668. body='Body 1',
  669. ),
  670. issue_type='issue',
  671. instruction='Test instruction 1',
  672. base_commit='def456',
  673. git_patch='Test patch 1',
  674. history=[],
  675. metrics={},
  676. success=True,
  677. comment_success=None,
  678. success_explanation='Test success 1',
  679. error=None,
  680. )
  681. # Mock return value
  682. mock_send_pull_request.return_value = (
  683. 'https://github.com/test-owner/test-repo/pull/1'
  684. )
  685. mock_initialize_repo.return_value = f'{mock_output_dir}/patches/issue_1'
  686. # Call the function
  687. process_single_issue(
  688. mock_output_dir,
  689. resolver_output,
  690. github_token,
  691. github_username,
  692. pr_type,
  693. mock_llm_config,
  694. None,
  695. False,
  696. None,
  697. )
  698. # Assert that the mocked functions were called with correct arguments
  699. mock_initialize_repo.assert_called_once_with(mock_output_dir, 1, 'issue', 'def456')
  700. mock_apply_patch.assert_called_once_with(
  701. f'{mock_output_dir}/patches/issue_1', resolver_output.git_patch
  702. )
  703. mock_make_commit.assert_called_once_with(
  704. f'{mock_output_dir}/patches/issue_1', resolver_output.issue, 'issue'
  705. )
  706. mock_send_pull_request.assert_called_once_with(
  707. github_issue=resolver_output.issue,
  708. github_token=github_token,
  709. github_username=github_username,
  710. patch_dir=f'{mock_output_dir}/patches/issue_1',
  711. pr_type=pr_type,
  712. fork_owner=None,
  713. additional_message=resolver_output.success_explanation,
  714. target_branch=None,
  715. reviewer=None,
  716. pr_title=None,
  717. )
  718. @patch('openhands.resolver.send_pull_request.initialize_repo')
  719. @patch('openhands.resolver.send_pull_request.apply_patch')
  720. @patch('openhands.resolver.send_pull_request.send_pull_request')
  721. @patch('openhands.resolver.send_pull_request.make_commit')
  722. def test_process_single_issue_unsuccessful(
  723. mock_make_commit,
  724. mock_send_pull_request,
  725. mock_apply_patch,
  726. mock_initialize_repo,
  727. mock_output_dir,
  728. mock_llm_config,
  729. ):
  730. # Initialize test data
  731. github_token = 'test_token'
  732. github_username = 'test_user'
  733. pr_type = 'draft'
  734. resolver_output = ResolverOutput(
  735. issue=GithubIssue(
  736. owner='test-owner',
  737. repo='test-repo',
  738. number=1,
  739. title='Issue 1',
  740. body='Body 1',
  741. ),
  742. issue_type='issue',
  743. instruction='Test instruction 1',
  744. base_commit='def456',
  745. git_patch='Test patch 1',
  746. history=[],
  747. metrics={},
  748. success=False,
  749. comment_success=None,
  750. success_explanation='',
  751. error='Test error',
  752. )
  753. # Call the function
  754. process_single_issue(
  755. mock_output_dir,
  756. resolver_output,
  757. github_token,
  758. github_username,
  759. pr_type,
  760. mock_llm_config,
  761. None,
  762. False,
  763. None,
  764. )
  765. # Assert that none of the mocked functions were called
  766. mock_initialize_repo.assert_not_called()
  767. mock_apply_patch.assert_not_called()
  768. mock_make_commit.assert_not_called()
  769. mock_send_pull_request.assert_not_called()
  770. @patch('openhands.resolver.send_pull_request.load_all_resolver_outputs')
  771. @patch('openhands.resolver.send_pull_request.process_single_issue')
  772. def test_process_all_successful_issues(
  773. mock_process_single_issue, mock_load_all_resolver_outputs, mock_llm_config
  774. ):
  775. # Create ResolverOutput objects with properly initialized GithubIssue instances
  776. resolver_output_1 = ResolverOutput(
  777. issue=GithubIssue(
  778. owner='test-owner',
  779. repo='test-repo',
  780. number=1,
  781. title='Issue 1',
  782. body='Body 1',
  783. ),
  784. issue_type='issue',
  785. instruction='Test instruction 1',
  786. base_commit='def456',
  787. git_patch='Test patch 1',
  788. history=[],
  789. metrics={},
  790. success=True,
  791. comment_success=None,
  792. success_explanation='Test success 1',
  793. error=None,
  794. )
  795. resolver_output_2 = ResolverOutput(
  796. issue=GithubIssue(
  797. owner='test-owner',
  798. repo='test-repo',
  799. number=2,
  800. title='Issue 2',
  801. body='Body 2',
  802. ),
  803. issue_type='issue',
  804. instruction='Test instruction 2',
  805. base_commit='ghi789',
  806. git_patch='Test patch 2',
  807. history=[],
  808. metrics={},
  809. success=False,
  810. comment_success=None,
  811. success_explanation='',
  812. error='Test error 2',
  813. )
  814. resolver_output_3 = ResolverOutput(
  815. issue=GithubIssue(
  816. owner='test-owner',
  817. repo='test-repo',
  818. number=3,
  819. title='Issue 3',
  820. body='Body 3',
  821. ),
  822. issue_type='issue',
  823. instruction='Test instruction 3',
  824. base_commit='jkl012',
  825. git_patch='Test patch 3',
  826. history=[],
  827. metrics={},
  828. success=True,
  829. comment_success=None,
  830. success_explanation='Test success 3',
  831. error=None,
  832. )
  833. mock_load_all_resolver_outputs.return_value = [
  834. resolver_output_1,
  835. resolver_output_2,
  836. resolver_output_3,
  837. ]
  838. # Call the function
  839. process_all_successful_issues(
  840. 'output_dir',
  841. 'github_token',
  842. 'github_username',
  843. 'draft',
  844. mock_llm_config, # llm_config
  845. None, # fork_owner
  846. )
  847. # Assert that process_single_issue was called for successful issues only
  848. assert mock_process_single_issue.call_count == 2
  849. # Check that the function was called with the correct arguments for successful issues
  850. mock_process_single_issue.assert_has_calls(
  851. [
  852. call(
  853. 'output_dir',
  854. resolver_output_1,
  855. 'github_token',
  856. 'github_username',
  857. 'draft',
  858. mock_llm_config,
  859. None,
  860. False,
  861. None,
  862. ),
  863. call(
  864. 'output_dir',
  865. resolver_output_3,
  866. 'github_token',
  867. 'github_username',
  868. 'draft',
  869. mock_llm_config,
  870. None,
  871. False,
  872. None,
  873. ),
  874. ]
  875. )
  876. # Add more assertions as needed to verify the behavior of the function
  877. @patch('requests.get')
  878. @patch('subprocess.run')
  879. def test_send_pull_request_branch_naming(
  880. mock_run, mock_get, mock_github_issue, mock_output_dir
  881. ):
  882. repo_path = os.path.join(mock_output_dir, 'repo')
  883. # Mock API responses
  884. mock_get.side_effect = [
  885. MagicMock(status_code=200), # First branch exists
  886. MagicMock(status_code=200), # Second branch exists
  887. MagicMock(status_code=404), # Third branch doesn't exist
  888. MagicMock(json=lambda: {'default_branch': 'main'}), # Get default branch
  889. ]
  890. # Mock subprocess.run calls
  891. mock_run.side_effect = [
  892. MagicMock(returncode=0), # git checkout -b
  893. MagicMock(returncode=0), # git push
  894. ]
  895. # Call the function
  896. result = send_pull_request(
  897. github_issue=mock_github_issue,
  898. github_token='test-token',
  899. github_username='test-user',
  900. patch_dir=repo_path,
  901. pr_type='branch',
  902. )
  903. # Assert API calls
  904. assert mock_get.call_count == 4
  905. # Check branch creation and push
  906. assert mock_run.call_count == 2
  907. checkout_call, push_call = mock_run.call_args_list
  908. assert checkout_call == call(
  909. ['git', '-C', repo_path, 'checkout', '-b', 'openhands-fix-issue-42-try3'],
  910. capture_output=True,
  911. text=True,
  912. )
  913. assert push_call == call(
  914. [
  915. 'git',
  916. '-C',
  917. repo_path,
  918. 'push',
  919. 'https://test-user:test-token@github.com/test-owner/test-repo.git',
  920. 'openhands-fix-issue-42-try3',
  921. ],
  922. capture_output=True,
  923. text=True,
  924. )
  925. # Check the result
  926. assert (
  927. result
  928. == 'https://github.com/test-owner/test-repo/compare/openhands-fix-issue-42-try3?expand=1'
  929. )
  930. @patch('openhands.resolver.send_pull_request.argparse.ArgumentParser')
  931. @patch('openhands.resolver.send_pull_request.process_all_successful_issues')
  932. @patch('openhands.resolver.send_pull_request.process_single_issue')
  933. @patch('openhands.resolver.send_pull_request.load_single_resolver_output')
  934. @patch('os.path.exists')
  935. @patch('os.getenv')
  936. def test_main(
  937. mock_getenv,
  938. mock_path_exists,
  939. mock_load_single_resolver_output,
  940. mock_process_single_issue,
  941. mock_process_all_successful_issues,
  942. mock_parser,
  943. ):
  944. from openhands.resolver.send_pull_request import main
  945. # Setup mock parser
  946. mock_args = MagicMock()
  947. mock_args.github_token = None
  948. mock_args.github_username = 'mock_username'
  949. mock_args.output_dir = '/mock/output'
  950. mock_args.pr_type = 'draft'
  951. mock_args.issue_number = '42'
  952. mock_args.fork_owner = None
  953. mock_args.send_on_failure = False
  954. mock_args.llm_model = 'mock_model'
  955. mock_args.llm_base_url = 'mock_url'
  956. mock_args.llm_api_key = 'mock_key'
  957. mock_args.target_branch = None
  958. mock_args.reviewer = None
  959. mock_args.pr_title = None
  960. mock_parser.return_value.parse_args.return_value = mock_args
  961. # Setup environment variables
  962. mock_getenv.side_effect = (
  963. lambda key, default=None: 'mock_token' if key == 'GITHUB_TOKEN' else default
  964. )
  965. # Setup path exists
  966. mock_path_exists.return_value = True
  967. # Setup mock resolver output
  968. mock_resolver_output = MagicMock()
  969. mock_load_single_resolver_output.return_value = mock_resolver_output
  970. # Run main function
  971. main()
  972. llm_config = LLMConfig(
  973. model=mock_args.llm_model,
  974. base_url=mock_args.llm_base_url,
  975. api_key=mock_args.llm_api_key,
  976. )
  977. # Use any_call instead of assert_called_with for more flexible matching
  978. assert mock_process_single_issue.call_args == call(
  979. '/mock/output',
  980. mock_resolver_output,
  981. 'mock_token',
  982. 'mock_username',
  983. 'draft',
  984. llm_config,
  985. None,
  986. False,
  987. mock_args.target_branch,
  988. mock_args.reviewer,
  989. mock_args.pr_title,
  990. )
  991. # Other assertions
  992. mock_parser.assert_called_once()
  993. mock_getenv.assert_any_call('GITHUB_TOKEN')
  994. mock_path_exists.assert_called_with('/mock/output')
  995. mock_load_single_resolver_output.assert_called_with('/mock/output/output.jsonl', 42)
  996. # Test for 'all_successful' issue number
  997. mock_args.issue_number = 'all_successful'
  998. main()
  999. mock_process_all_successful_issues.assert_called_with(
  1000. '/mock/output',
  1001. 'mock_token',
  1002. 'mock_username',
  1003. 'draft',
  1004. llm_config,
  1005. None,
  1006. )
  1007. # Test for invalid issue number
  1008. mock_args.issue_number = 'invalid'
  1009. with pytest.raises(ValueError):
  1010. main()
  1011. @patch('subprocess.run')
  1012. def test_make_commit_escapes_issue_title(mock_subprocess_run):
  1013. # Setup
  1014. repo_dir = '/path/to/repo'
  1015. issue = GithubIssue(
  1016. owner='test-owner',
  1017. repo='test-repo',
  1018. number=42,
  1019. title='Issue with "quotes" and $pecial characters',
  1020. body='Test body',
  1021. )
  1022. # Mock subprocess.run to return success for all calls
  1023. mock_subprocess_run.return_value = MagicMock(
  1024. returncode=0, stdout='sample output', stderr=''
  1025. )
  1026. # Call the function
  1027. issue_type = 'issue'
  1028. make_commit(repo_dir, issue, issue_type)
  1029. # Assert that subprocess.run was called with the correct arguments
  1030. calls = mock_subprocess_run.call_args_list
  1031. assert len(calls) == 4 # git config check, git add, git commit
  1032. # Check the git commit call
  1033. git_commit_call = calls[3][0][0]
  1034. expected_commit_message = (
  1035. 'Fix issue #42: Issue with "quotes" and $pecial characters'
  1036. )
  1037. assert [
  1038. 'git',
  1039. '-C',
  1040. '/path/to/repo',
  1041. 'commit',
  1042. '-m',
  1043. expected_commit_message,
  1044. ] == git_commit_call
  1045. @patch('subprocess.run')
  1046. def test_make_commit_no_changes(mock_subprocess_run):
  1047. # Setup
  1048. repo_dir = '/path/to/repo'
  1049. issue = GithubIssue(
  1050. owner='test-owner',
  1051. repo='test-repo',
  1052. number=42,
  1053. title='Issue with no changes',
  1054. body='Test body',
  1055. )
  1056. # Mock subprocess.run to simulate no changes in the repo
  1057. mock_subprocess_run.side_effect = [
  1058. MagicMock(returncode=0),
  1059. MagicMock(returncode=0),
  1060. MagicMock(returncode=1, stdout=''), # git status --porcelain (no changes)
  1061. ]
  1062. with pytest.raises(
  1063. RuntimeError, match='ERROR: Openhands failed to make code changes.'
  1064. ):
  1065. make_commit(repo_dir, issue, 'issue')
  1066. # Check that subprocess.run was called for checking git status and add, but not commit
  1067. assert mock_subprocess_run.call_count == 3
  1068. git_status_call = mock_subprocess_run.call_args_list[2][0][0]
  1069. assert f'git -C {repo_dir} status --porcelain' in git_status_call
  1070. def test_apply_patch_rename_directory(mock_output_dir):
  1071. # Create a sample directory structure
  1072. old_dir = os.path.join(mock_output_dir, 'prompts', 'resolve')
  1073. os.makedirs(old_dir)
  1074. # Create test files
  1075. test_files = [
  1076. 'issue-success-check.jinja',
  1077. 'pr-feedback-check.jinja',
  1078. 'pr-thread-check.jinja',
  1079. ]
  1080. for filename in test_files:
  1081. file_path = os.path.join(old_dir, filename)
  1082. with open(file_path, 'w') as f:
  1083. f.write(f'Content of {filename}')
  1084. # Create a patch that renames the directory
  1085. patch_content = """diff --git a/prompts/resolve/issue-success-check.jinja b/prompts/guess_success/issue-success-check.jinja
  1086. similarity index 100%
  1087. rename from prompts/resolve/issue-success-check.jinja
  1088. rename to prompts/guess_success/issue-success-check.jinja
  1089. diff --git a/prompts/resolve/pr-feedback-check.jinja b/prompts/guess_success/pr-feedback-check.jinja
  1090. similarity index 100%
  1091. rename from prompts/resolve/pr-feedback-check.jinja
  1092. rename to prompts/guess_success/pr-feedback-check.jinja
  1093. diff --git a/prompts/resolve/pr-thread-check.jinja b/prompts/guess_success/pr-thread-check.jinja
  1094. similarity index 100%
  1095. rename from prompts/resolve/pr-thread-check.jinja
  1096. rename to prompts/guess_success/pr-thread-check.jinja"""
  1097. # Apply the patch
  1098. apply_patch(mock_output_dir, patch_content)
  1099. # Check if files were moved correctly
  1100. new_dir = os.path.join(mock_output_dir, 'prompts', 'guess_success')
  1101. assert not os.path.exists(old_dir), 'Old directory still exists'
  1102. assert os.path.exists(new_dir), 'New directory was not created'
  1103. # Check if all files were moved and content preserved
  1104. for filename in test_files:
  1105. old_path = os.path.join(old_dir, filename)
  1106. new_path = os.path.join(new_dir, filename)
  1107. assert not os.path.exists(old_path), f'Old file {filename} still exists'
  1108. assert os.path.exists(new_path), f'New file {filename} was not created'
  1109. with open(new_path, 'r') as f:
  1110. content = f.read()
  1111. assert content == f'Content of {filename}', f'Content mismatch for {filename}'