├── .gitignore ├── LICENSE ├── README.md ├── azure.py ├── claude.py ├── claude └── claude-v1.3.jsonl ├── data ├── HumanEval.jsonl ├── HumanEval.jsonl.gz ├── example_problem.jsonl ├── example_samples.jsonl ├── example_samples.jsonl_results.jsonl ├── results-code-cushman-001-1.jsonl ├── results-code-cushman-001-10.jsonl └── results-code-davinci-002-1.jsonl ├── gpt4 ├── results-314.jsonl └── results-531.jsonl ├── gpt_3_5 ├── azure.jsonl └── azure.jsonl_results.jsonl ├── human_eval ├── __init__.py ├── data.py ├── evaluate_functional_correctness.py ├── evaluation.py └── execution.py ├── prompt_utils.py ├── prompts.py ├── requirements.txt ├── run.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | results/* 2 | .env 3 | __pycache__/ 4 | 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) OpenAI (https://openai.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### HumanEval for GPT-3.5/GPT-4 2 | 3 | Results are [here](https://twitter.com/amanrsanger/status/1635751764577361921). Forked from [OpenAI's repo](https://github.com/openai/human-eval). 4 | 5 | To generate the completions (after pip installing the requirements), run: 6 | ``` 7 | mkdir results 8 | python run.py 9 | ``` 10 | 11 | Then to evaluate the completion results, run 12 | ``` 13 | pip3 install -e . 14 | evaluate_functional_correctness results/name_of_results_file 15 | ``` 16 | -------------------------------------------------------------------------------- /azure.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | from aiohttp import ClientSession 4 | import os 5 | from dotenv import load_dotenv 6 | 7 | load_dotenv() 8 | 9 | 10 | async def azure_complete(prompt: str): 11 | azure_openai_base = os.environ.get('azureOpenaiBase') 12 | azure_openai_deployment = os.environ.get('azureOpenaiDeployment') 13 | azure_openai_key = os.environ.get('azureOpenaiKey') 14 | 15 | api_version = "2023-03-15-preview" 16 | model = "GPT_3_5_TURBO" 17 | 18 | url = f"{azure_openai_base}openai/deployments/{azure_openai_deployment}/completions?api-version={api_version}" 19 | headers = { 20 | "api-key": azure_openai_key, 21 | } 22 | json_data = { 23 | "prompt": prompt, 24 | "temperature": 0, 25 | "max_tokens": 1200, 26 | "stream": False, 27 | "model": model, 28 | "stop": '```' 29 | } 30 | 31 | async with ClientSession() as session: 32 | async with session.post(url, json=json_data, headers=headers, timeout=5.0) as response: 33 | if response.status != 200: 34 | raise Exception(f"Request failed with status {response.status}") 35 | 36 | return await response.json() 37 | -------------------------------------------------------------------------------- /claude.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import os 3 | 4 | ANTHROPIC_HEADERS = { 5 | "X-API-Key": os.environ['ANTHROPIC_API_KEY'], 6 | "Content-Type": "application/json", 7 | } 8 | 9 | async def claude_complete(prompt: str, model: str, temperature=0.0): 10 | async with aiohttp.ClientSession() as session: 11 | print('Running claude') 12 | response = await session.post('https://api.anthropic.com/v1/complete', 13 | headers=ANTHROPIC_HEADERS, 14 | json={ 15 | "prompt": prompt, 16 | "model": model, 17 | "max_tokens_to_sample": 500, 18 | "temperature": temperature, 19 | # "n": num_tries, 20 | "stream": False, 21 | "stop": "```" 22 | # "logprobs": None, 23 | }) 24 | data = await response.json() 25 | # print(repr(data)) 26 | completion = data['completion'] 27 | if completion.endswith('```'): 28 | completion = completion[:-3] 29 | print(prompt) 30 | print('_____') 31 | print(completion) 32 | print('\n\n\n') 33 | 34 | return [completion] -------------------------------------------------------------------------------- /claude/claude-v1.3.jsonl: -------------------------------------------------------------------------------- 1 | {"task_id": "HumanEval/82", "completion": "\n length = len(string)\n if length <= 1:\n return False\n for i in range(2, length):\n if length % i == 0:\n return False\n return True\n"} 2 | {"task_id": "HumanEval/9", "completion": "\n result = []\n max_so_far = numbers[0]\n result.append(max_so_far)\n for num in numbers[1:]:\n max_so_far = max(max_so_far, num)\n result.append(max_so_far)\n return result"} 3 | {"task_id": "HumanEval/155", "completion": "\n even_count = 0\n odd_count = 0\n \n while num != 0:\n digit = num % 10\n if digit % 2 == 0:\n even_count += 1\n else:\n odd_count += 1\n num //= 10\n \n return (even_count, odd_count)\n"} 4 | {"task_id": "HumanEval/154", "completion": "\n for i in range(len(b)):\n if a.find(b[i:] + b[:i]) != -1:\n return True\n return False"} 5 | {"task_id": "HumanEval/11", "completion": "\n result = ''\n for i in range(len(a)):\n if a[i] == b[i]:\n result += '0'\n else:\n result += '1'\n return result\n"} 6 | {"task_id": "HumanEval/85", "completion": "\n sum = 0\n for i in range(len(lst)):\n if i % 2 != 0 and lst[i] % 2 == 0:\n sum += lst[i]\n return sum\n"} 7 | {"task_id": "HumanEval/10", "completion": "\n if len(string) == 0:\n return ''\n for i in range(len(string), -1, -1):\n postfix = string[i:]\n if is_palindrome(postfix):\n prefix = string[:i]\n return prefix + prefix[::-1]"} 8 | {"task_id": "HumanEval/13", "completion": "\n while b != 0:\n a, b = b, a % b\n return a"} 9 | {"task_id": "HumanEval/157", "completion": "\n if a**2 + b**2 == c**2:\n return True\n else:\n return False"} 10 | {"task_id": "HumanEval/83", "completion": "\n if n == 1:\n return 10\n count = 0\n for i in range(1, 10):\n count += 9 * (10 ** (n-2)) # Numbers with 1 at the end\n if i == 1:\n count += 10 ** (n-1) # Numbers with 1 at the beginning\n return count"} 11 | {"task_id": "HumanEval/15", "completion": "\n result = \"\"\n for i in range(n + 1):\n result += str(i) + \" \"\n return result.strip()\n"} 12 | {"task_id": "HumanEval/159", "completion": "\n total_eaten = number\n if need <= remaining:\n total_eaten += need\n remaining -= need\n else:\n total_eaten += remaining\n remaining = 0\n return [total_eaten, remaining]\n"} 13 | {"task_id": "HumanEval/87", "completion": "\n result = []\n for row in range(len(lst)):\n if x in lst[row]:\n col = lst[row].index(x)\n result.append((row, col))\n result.sort()\n for row_tup in result:\n lst[row_tup[0]].sort(reverse=True)\n return result\n"} 14 | {"task_id": "HumanEval/89", "completion": "\n alphabet = 'abcdefghijklmnopqrstuvwxyz'\n new_alphabet = alphabet[2:] + alphabet[:2]\n translation = str.maketrans(alphabet, new_alphabet)\n return s.translate(translation)\n"} 15 | {"task_id": "HumanEval/163", "completion": "\n result = []\n for num in range(a, b + 1):\n if num % 2 == 0:\n result.append(num)\n return sorted(result)\n"} 16 | {"task_id": "HumanEval/17", "completion": "\n notes = {\n 'o': 4,\n 'o|': 2,\n '.|': 1\n }\n\n result = []\n i = 0\n while i < len(music_string):\n if music_string[i] == 'o':\n result.append(notes['o'])\n i += 1\n elif music_string[i:i+2] == 'o|':\n result.append(notes['o|'])\n i += 2\n elif music_string[i:i+2] == '.|':\n result.append(notes['.|'])\n i += 2\n return result\n"} 17 | {"task_id": "HumanEval/91", "completion": "\n boredom_count = 0\n sentences = S.split('. ') + S.split('? ') + S.split('! ')\n for sentence in sentences:\n if sentence.startswith('I '):\n boredom_count += 1\n return boredom_count"} 18 | {"task_id": "HumanEval/20", "completion": "\n min_diff = float('inf')\n closest_pair = (0, 0)\n numbers.sort()\n for i in range(len(numbers)-1):\n diff = abs(numbers[i] - numbers[i+1])\n if diff < min_diff:\n min_diff = diff\n closest_pair = (numbers[i], numbers[i+1])\n return closest_pair\n"} 19 | {"task_id": "HumanEval/22", "completion": "\n return [x for x in values if isinstance(x, int)]\n"} 20 | {"task_id": "HumanEval/161", "completion": "\n result = \"\"\n for char in s:\n if char.isalpha():\n result += char.swapcase()\n else:\n result += char\n if not any(c.isalpha() for c in s):\n result = result[::-1]\n return result"} 21 | {"task_id": "HumanEval/21", "completion": "\n minimum = min(numbers)\n maximum = max(numbers)\n range = maximum - minimum\n return [(num - minimum) / range for num in numbers]\n"} 22 | {"task_id": "HumanEval/23", "completion": "\n return len(string)\n"} 23 | {"task_id": "HumanEval/26", "completion": "\n seen = set()\n result = []\n for num in numbers:\n if num not in seen:\n result.append(num)\n seen.add(num)\n return result\n"} 24 | {"task_id": "HumanEval/19", "completion": "\n num_words = {'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight': 8, 'nine': 9}\n nums = [num_words[word] for word in numbers.split()]\n nums.sort()\n return ' '.join([list(num_words.keys())[list(num_words.values()).index(num)] for num in nums])"} 25 | {"task_id": "HumanEval/24", "completion": "\n for i in range(n-1, 1, -1):\n if n % i == 0:\n return i"} 26 | {"task_id": "HumanEval/27", "completion": "\n result = ''\n for char in string:\n if char.isupper():\n result += char.lower()\n elif char.islower():\n result += char.upper()\n else:\n result += char\n return result\n"} 27 | {"task_id": "HumanEval/28", "completion": "\n result = ''\n for string in strings:\n result += string\n return result\n"} 28 | {"task_id": "HumanEval/29", "completion": "\n return [string for string in strings if string.startswith(prefix)]\n"} 29 | {"task_id": "HumanEval/30", "completion": "\n positive_numbers = []\n for num in l:\n if num > 0:\n positive_numbers.append(num)\n return positive_numbers\n"} 30 | {"task_id": "HumanEval/25", "completion": "\n factors = []\n divisor = 2\n while divisor <= n:\n if n % divisor == 0:\n factors.append(divisor)\n n = n / divisor\n else:\n divisor += 1\n return factors"} 31 | {"task_id": "HumanEval/31", "completion": "\n if n <= 1:\n return False\n for i in range(2, n):\n if n % i == 0:\n return False\n return True\n"} 32 | {"task_id": "HumanEval/34", "completion": "\n return sorted(set(l))\n"} 33 | {"task_id": "HumanEval/35", "completion": "\n max_elem = l[0]\n for elem in l:\n if elem > max_elem:\n max_elem = elem\n return max_elem\n"} 34 | {"task_id": "HumanEval/36", "completion": "\n count = 0\n for i in range(1, n):\n if i % 11 == 0 or i % 13 == 0:\n if '7' in str(i):\n count += 1\n return count\n"} 35 | {"task_id": "HumanEval/33", "completion": "\n l_copy = l.copy()\n for i in range(2, len(l), 3):\n l_copy[i:i+3] = sorted(l[i:i+3])\n return l_copy"} 36 | {"task_id": "HumanEval/32", "completion": "\n n = len(xs)\n x0 = 0\n x1 = 1\n while poly(xs, x0) * poly(xs, x1) > 0:\n x0 = x1\n x1 *= 2\n while abs(x1 - x0) > 1e-6:\n x2 = (x0 + x1) / 2\n if poly(xs, x0) * poly(xs, x2) < 0:\n x1 = x2\n else:\n x0 = x2\n return x2\n"} 37 | {"task_id": "HumanEval/38", "completion": "\n # split string to groups. Each of length 3.\n groups = [s[(3 * i):min((3 * i + 3), len(s))] for i in range((len(s) + 2) // 3)]\n # reverse cycle elements in each group. Unless group has fewer elements than 3.\n groups = [(group[-1] + group[:-1]) if len(group) == 3 else group for group in groups]\n return \"\".join(groups)\n"} 38 | {"task_id": "HumanEval/37", "completion": "\n even_indices = [l[i] for i in range(len(l)) if i % 2 == 0]\n even_indices.sort()\n result = []\n for i in range(len(l)):\n if i % 2 == 0:\n result.append(even_indices.pop(0))\n else:\n result.append(l[i])\n return result"} 39 | {"task_id": "HumanEval/40", "completion": "\n seen = set()\n for num in l:\n if -num in seen:\n return True\n seen.add(num)\n return False"} 40 | {"task_id": "HumanEval/115", "completion": "\n rows, cols = len(grid), len(grid[0])\n total = 0\n for r in range(rows):\n count = 0\n for c in range(cols):\n if grid[r][c] == 1:\n count += 1\n total += math.ceil(count / capacity)\n return total\n"} 41 | {"task_id": "HumanEval/39", "completion": "\n fib_num = 1\n prime_count = 1\n while prime_count <= n:\n fib_num = fib(fib_num)\n if is_prime(fib_num):\n prime_count += 1\n return fib_num\n"} 42 | {"task_id": "HumanEval/41", "completion": "\n collisions = 0\n for i in range(n):\n collisions += i\n return collisions"} 43 | {"task_id": "HumanEval/118", "completion": "\n vowels = ['a', 'e', 'i', 'o', 'u']\n for i in range(len(word)-1, 0, -1):\n if word[i] in vowels and word[i-1] not in vowels and word[i+1] not in vowels:\n return word[i]\n return ''\n"} 44 | {"task_id": "HumanEval/117", "completion": "\n consonants = set('bcdfghjklmnpqrstvwxyz')\n words = s.lower().split()\n result = []\n for word in words:\n count = 0\n for char in word:\n if char in consonants:\n count += 1\n if count == n:\n result.append(word)\n return result"} 45 | {"task_id": "HumanEval/116", "completion": "\n def count_ones(n):\n count = 0\n while n:\n count += n & 1\n n >>= 1\n return count\n \n arr.sort(key=lambda x: (count_ones(x), x))\n return arr"} 46 | {"task_id": "HumanEval/121", "completion": "\n sum = 0\n for i in range(len(lst)):\n if i % 2 == 0 and lst[i] % 2 == 1:\n sum += lst[i]\n return sum\n"} 47 | {"task_id": "HumanEval/122", "completion": "\n sum = 0\n for num in arr[:k]:\n if num <= 99:\n sum += num\n return sum\n"} 48 | {"task_id": "HumanEval/120", "completion": "\n arr.sort()\n return arr[-k:]"} 49 | {"task_id": "HumanEval/119", "completion": "\n s1, s2 = lst\n stack = []\n for c in s1:\n if c == '(':\n stack.append(c)\n else:\n if not stack:\n return 'No'\n stack.pop()\n for c in s2:\n if c == '(':\n stack.append(c)\n else:\n if not stack:\n return 'No'\n stack.pop()\n return 'Yes' if not stack else 'No'"} 50 | {"task_id": "HumanEval/123", "completion": "\n sequence = [n]\n odd_numbers = []\n while n > 1:\n if n % 2 == 0:\n n = n // 2\n else:\n n = 3 * n + 1\n sequence.append(n)\n if n % 2 != 0:\n odd_numbers.append(n)\n odd_numbers.sort()\n return odd_numbers"} 51 | {"task_id": "HumanEval/126", "completion": "\n if len(lst) <= 1:\n return True\n prev = lst[0]\n count = 1\n for num in lst[1:]:\n if num == prev:\n count += 1\n if count > 1:\n return False\n elif num > prev:\n prev = num\n count = 1\n else:\n return False\n return True\n"} 52 | {"task_id": "HumanEval/125", "completion": "\n if ' ' in txt:\n return txt.split(' ')\n elif ',' in txt:\n return txt.split(',')\n else:\n count = 0\n for char in txt:\n if ord(char) % 2 == 1 and ord(char) <= ord('z'):\n count += 1\n return count\n"} 53 | {"task_id": "HumanEval/124", "completion": "\n if date == '':\n return False\n \n date_split = date.split('-')\n if len(date_split) != 3:\n return False\n \n month, day, year = date_split\n if not month.isdigit() or not day.isdigit() or not year.isdigit():\n return False\n \n month = int(month)\n day = int(day)\n year = int(year)\n \n if month < 1 or month > 12:\n return False\n \n if month in [1, 3, 5, 7, 8, 10, 12] and (day < 1 or day > 31):\n return False\n if month in [4, 6, 9, 11] and (day < 1 or day > 30):\n return False\n if month == 2 and (day < 1 or day > 29):\n return False\n \n return True\n"} 54 | {"task_id": "HumanEval/127", "completion": "\n start1, end1 = interval1\n start2, end2 = interval2\n \n # Check if intervals intersect\n if start1 > end2 or start2 > end1:\n return \"NO\"\n \n # Find intersection\n start = max(start1, start2)\n end = min(end1, end2)\n \n # Check if length is prime\n length = end - start\n if length == 2 or length == 3 or length == 5 or length == 7:\n return \"YES\"\n else:\n return \"NO\"\n"} 55 | {"task_id": "HumanEval/129", "completion": "\n # Your code here\n"} 56 | {"task_id": "HumanEval/133", "completion": "\n sum = 0\n for num in lst:\n sum += math.ceil(num)**2\n return sum\n"} 57 | {"task_id": "HumanEval/130", "completion": "\n if n == 0:\n return [1]\n if n == 1:\n return [1, 3]\n if n == 2:\n return [1, 3, 2]\n if n % 2 == 0:\n return [1, 3, 2] + [1 + n//2] + tri(n-1)\n else:\n return [1, 3, 2] + tri(n-1) + tri(n-2) + tri(n+1) "} 58 | {"task_id": "HumanEval/131", "completion": "\n product = 1\n while n > 0:\n digit = n % 10\n if digit % 2 == 1:\n product *= digit\n n //= 10\n return product"} 59 | {"task_id": "HumanEval/132", "completion": "\n stack = []\n for char in string:\n if char == '[':\n stack.append(char)\n else:\n if not stack:\n return False\n stack.pop()\n return len(stack) > 0"} 60 | {"task_id": "HumanEval/128", "completion": "\n if not arr:\n return None\n \n sign_prod = 1\n sum_mag = 0\n \n for num in arr:\n if num > 0:\n sign_prod *= 1\n sum_mag += num\n elif num < 0:\n sign_prod *= -1\n sum_mag += abs(num)\n else:\n sign_prod *= 0\n \n return sum_mag * sign_prod"} 61 | {"task_id": "HumanEval/135", "completion": "\n for i in range(len(arr)-1, 0, -1):\n if arr[i] < arr[i-1]:\n return i\n return -1"} 62 | {"task_id": "HumanEval/134", "completion": "\n if not txt:\n return False\n last_char = txt[-1]\n if last_char.isalpha() and txt[-2] == ' ':\n return True\n else:\n return False"} 63 | {"task_id": "HumanEval/136", "completion": "\n largest_negative = None\n smallest_positive = None\n \n for num in lst:\n if num < 0 and (largest_negative is None or num > largest_negative):\n largest_negative = num\n elif num > 0 and (smallest_positive is None or num < smallest_positive):\n smallest_positive = num\n \n return (largest_negative, smallest_positive)"} 64 | {"task_id": "HumanEval/139", "completion": "\n result = 1\n for i in range(1, n + 1):\n result *= i * special_factorial(i-1)\n return result\n"} 65 | {"task_id": "HumanEval/137", "completion": "\n if type(a) != type(b):\n return None\n if type(a) == int or type(a) == float:\n if a > b:\n return a\n elif b > a:\n return b\n else:\n return None\n if type(a) == str:\n a = a.replace(',', '.')\n b = b.replace(',', '.')\n a = float(a)\n b = float(b)\n if a > b:\n return str(a)\n elif b > a:\n return str(b)\n else:\n return None\n"} 66 | {"task_id": "HumanEval/138", "completion": "\n if n % 2 == 1:\n return False\n for a in range(2, n//2 + 1, 2):\n for b in range(2, n//2 + 1, 2):\n for c in range(2, n//2 + 1, 2):\n if a + b + c + (n - a - b - c) == n:\n return True\n return False"} 67 | {"task_id": "HumanEval/141", "completion": "\n digits = 0\n for char in file_name:\n if char.isdigit():\n digits += 1\n if digits > 3:\n return 'No'\n if file_name.count('.') != 1:\n return 'No'\n split_name = file_name.split('.')\n if not split_name[0] or not split_name[0][0].isalpha():\n return 'No'\n if split_name[1] not in ['txt', 'exe', 'dll']:\n return 'No'\n return 'Yes'\n"} 68 | {"task_id": "HumanEval/142", "completion": "\n sum = 0\n for i in range(len(lst)):\n if i % 3 == 0:\n sum += lst[i] ** 2\n elif i % 4 == 0 and i % 3 != 0:\n sum += lst[i] ** 3\n else:\n sum += lst[i]\n return sum"} 69 | {"task_id": "HumanEval/143", "completion": "\n prime_words = []\n words = sentence.split()\n for word in words:\n if is_prime(len(word)):\n prime_words.append(word)\n return ' '.join(prime_words)\n"} 70 | {"task_id": "HumanEval/140", "completion": "\n result = \"\"\n space_count = 0\n for char in text:\n if char == \" \":\n space_count += 1\n if space_count > 2:\n result += \"-\"\n space_count = 0\n else:\n result += \"_\"\n else:\n result += char\n space_count = 0\n return result"} 71 | {"task_id": "HumanEval/74", "completion": "\n total1 = 0\n total2 = 0\n \n for string in lst1:\n total1 += len(string)\n for string in lst2:\n total2 += len(string)\n \n if total1 < total2:\n return lst1\n elif total2 < total1:\n return lst2\n else:\n return lst1\n"} 72 | {"task_id": "HumanEval/144", "completion": "\n x_num, x_den = x.split('/')\n n_num, n_den = n.split('/')\n x_num, x_den, n_num, n_den = int(x_num), int(x_den), int(n_num), int(n_den)\n \n if x_den == n_den:\n return x_num * n_num == x_den\n else:\n return x_num * n_den == x_den * n_num"} 73 | {"task_id": "HumanEval/146", "completion": "\n count = 0\n for num in nums:\n if num > 10:\n first_digit = num // 10\n last_digit = num % 10\n if first_digit % 2 == 1 and last_digit % 2 == 1:\n count += 1\n return count"} 74 | {"task_id": "HumanEval/3", "completion": "\n balance = 0\n for operation in operations:\n balance += operation\n if balance < 0:\n return True\n return False\n"} 75 | {"task_id": "HumanEval/145", "completion": "\n if not nums:\n return []\n \n sums = []\n for num in nums:\n sum_digits = 0\n while num:\n sum_digits += num % 10\n num //= 10\n sums.append(sum_digits)\n \n sorted_sums = sorted(sums)\n result = []\n for sum_digit in sorted_sums:\n for i in range(len(sums)):\n if sums[i] == sum_digit:\n result.append(nums[i])\n sums[i] = -1\n break\n return result"} 76 | {"task_id": "HumanEval/1", "completion": "\n groups = []\n current_group = ''\n open_parens = 0\n \n for char in paren_string:\n if char == '(':\n open_parens += 1\n current_group += char\n elif char == ')':\n open_parens -= 1\n current_group += char\n \n if open_parens == 0:\n groups.append(current_group)\n current_group = ''\n else:\n current_group += char\n \n return groups"} 77 | {"task_id": "HumanEval/5", "completion": "\n result = []\n for i in range(len(numbers)-1):\n result.append(numbers[i])\n result.append(delimeter)\n result.append(numbers[-1])\n return result\n"} 78 | {"task_id": "HumanEval/76", "completion": "\n if x == 1:\n return True\n power = 1\n while power < x:\n power = power * n\n if power == x:\n return True\n return False"} 79 | {"task_id": "HumanEval/7", "completion": "\n filtered = []\n for string in strings:\n if substring in string:\n filtered.append(string)\n return filtered\n"} 80 | {"task_id": "HumanEval/150", "completion": "\n if n <= 1:\n return y\n if n == 2:\n return x\n if n > 2 and n % 2 == 0:\n return y\n max_div = n ** 0.5\n for i in range(3, int(max_div)+1, 2):\n if n % i == 0:\n return y\n return x"} 81 | {"task_id": "HumanEval/78", "completion": "\n prime_digits = {'2', '3', '5', '7', 'B', 'D'}\n count = 0\n for char in num:\n if char in prime_digits:\n count += 1\n return count\n"} 82 | {"task_id": "HumanEval/148", "completion": "\n planets = ('Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Saturn', 'Uranus', 'Neptune')\n try:\n start = planets.index(planet1)\n end = planets.index(planet2)\n except ValueError:\n return ()\n \n return planets[start+1:end]"} 83 | {"task_id": "HumanEval/152", "completion": "\n result = []\n for i in range(len(game)):\n diff = abs(game[i] - guess[i])\n result.append(diff)\n return result"} 84 | {"task_id": "HumanEval/14", "completion": "\n prefixes = []\n for i in range(len(string)):\n prefixes.append(string[:i+1])\n return prefixes"} 85 | {"task_id": "HumanEval/86", "completion": "\n words = s.split()\n ordered_words = []\n for word in words:\n ordered_word = ''.join(sorted(word))\n ordered_words.append(ordered_word)\n return ' '.join(ordered_words)"} 86 | {"task_id": "HumanEval/81", "completion": "\n letter_grades = []\n for grade in grades:\n if grade >= 4.0:\n letter_grades.append('A+')\n elif grade > 3.7:\n letter_grades.append('A')\n elif grade > 3.3:\n letter_grades.append('A-')\n elif grade > 3.0:\n letter_grades.append('B+')\n elif grade > 2.7:\n letter_grades.append('B')\n elif grade > 2.3:\n letter_grades.append('B-')\n elif grade > 2.0:\n letter_grades.append('C+')\n elif grade > 1.7:\n letter_grades.append('C')\n elif grade > 1.3:\n letter_grades.append('C-')\n elif grade > 1.0:\n letter_grades.append('D+')\n elif grade > 0.7:\n letter_grades.append('D')\n elif grade > 0.0:\n letter_grades.append('D-')\n else:\n letter_grades.append('E')\n return letter_grades\n"} 87 | {"task_id": "HumanEval/80", "completion": "\n if len(s) < 3:\n return False\n \n for i in range(len(s)-2):\n if s[i] == s[i+1] or s[i+1] == s[i+2] or s[i] == s[i+2]:\n return False\n return True"} 88 | {"task_id": "HumanEval/156", "completion": "\n roman_numerals = {1: 'i', 4: 'iv', 5: 'v', 9: 'ix', 10: 'x', 40: 'xl', 50: 'l', 90: 'xc', 100: 'c', 400: 'cd', 500: 'd', 900: 'cm', 1000: 'm'}\n result = ''\n for value, numeral in roman_numerals.items():\n while number >= value:\n result += numeral\n number -= value\n return result"} 89 | {"task_id": "HumanEval/18", "completion": "\n count = 0\n for i in range(len(string)-len(substring)+1):\n if string[i:i+len(substring)] == substring:\n count += 1\n return count"} 90 | {"task_id": "HumanEval/160", "completion": "\n result = operand[0]\n for i in range(len(operator)):\n if operator[i] == '+':\n result += operand[i+1]\n elif operator[i] == '-':\n result -= operand[i+1]\n elif operator[i] == '*':\n result *= operand[i+1]\n elif operator[i] == '//':\n result //= operand[i+1]\n elif operator[i] == '**':\n result **= operand[i+1]\n return result\n"} 91 | {"task_id": "HumanEval/90", "completion": "\n if len(lst) < 2:\n return None\n smallest = lst[0]\n second_smallest = lst[1]\n for num in lst[2:]:\n if num < smallest:\n second_smallest = smallest\n smallest = num\n elif num < second_smallest:\n second_smallest = num\n return second_smallest"} 92 | {"task_id": "HumanEval/95", "completion": "\n if not dict:\n return False\n \n keys = dict.keys()\n all_lower = True\n all_upper = True\n \n for key in keys:\n if not key.islower():\n all_lower = False\n if not key.isupper():\n all_upper = False\n \n return all_lower or all_upper\n"} 93 | {"task_id": "HumanEval/97", "completion": "\n return int(str(a)[-1]) * int(str(b)[-1])\n"} 94 | {"task_id": "HumanEval/101", "completion": "\n return s.split()\n"} 95 | {"task_id": "HumanEval/99", "completion": "\n num = float(value)\n if num.is_integer():\n return int(num)\n else:\n rounded = round(num)\n if num - rounded > 0.5:\n return rounded + 1\n else:\n return rounded\n"} 96 | {"task_id": "HumanEval/93", "completion": "\n vowels = ['a', 'e', 'i', 'o', 'u']\n mapping = {'a': 'c', 'e': 'g', 'i': 'k', 'o': 'q', 'u': 'w'}\n \n encoded = ''\n for char in message:\n if char.lower() in vowels:\n encoded += mapping[char.lower()]\n else:\n if char.islower():\n encoded += char.upper()\n else:\n encoded += char.lower()\n return encoded"} 97 | {"task_id": "HumanEval/105", "completion": "\n nums = []\n for num in arr:\n if 1 <= num <= 9:\n nums.append(num)\n nums.sort()\n nums.reverse()\n \n names = [\"One\", \"Two\", \"Three\", \"Four\", \"Five\", \"Six\", \"Seven\", \"Eight\", \"Nine\"]\n return [names[num-1] for num in nums]\n"} 98 | {"task_id": "HumanEval/107", "completion": "\n even_count = 0\n odd_count = 0\n for num in range(1, n + 1):\n if str(num) == str(num)[::-1]:\n if num % 2 == 0:\n even_count += 1\n else:\n odd_count += 1\n return (even_count, odd_count)"} 99 | {"task_id": "HumanEval/103", "completion": "\n if n > m:\n return -1\n total = 0\n for i in range(n, m+1):\n total += i\n average = total / (m - n + 1)\n rounded = round(average)\n binary = bin(rounded)\n return binary"} 100 | {"task_id": "HumanEval/113", "completion": "\n result = []\n for num_str in lst:\n odd_count = 0\n for char in num_str:\n if int(char) % 2 == 1:\n odd_count += 1\n result.append(\"the number of odd elements \" + str(odd_count) + \"n the str\" + str(odd_count) + \"ng \" + str(odd_count) + \" of the \" + str(odd_count) + \"nput.\")\n return result\n"} 101 | {"task_id": "HumanEval/109", "completion": "\n if not arr:\n return True\n \n n = len(arr)\n i = 0\n while i < n-1 and arr[i] <= arr[i+1]:\n i += 1\n \n if i == n-1:\n return True\n \n j = i+1\n while j < n and arr[i] > arr[j]:\n j += 1\n \n if j == n:\n return False\n \n k = j\n while k < n and arr[i] <= arr[k]:\n k += 1\n \n if k == n:\n return True\n \n l = k \n while l < n and arr[j-1] > arr[l]:\n l += 1\n \n if l == n:\n return False\n \n return True \n"} 102 | {"task_id": "HumanEval/45", "completion": "\n return (a * h) / 2\n"} 103 | {"task_id": "HumanEval/43", "completion": "\n seen = set()\n for num in l:\n if -num in seen:\n return True\n seen.add(num)\n return False"} 104 | {"task_id": "HumanEval/111", "completion": "\n histogram = {}\n for char in test.lower().split(' '):\n if char in histogram:\n histogram[char] += 1\n else:\n histogram[char] = 1\n max_count = max(histogram.values())\n result = {k:v for k, v in histogram.items() if v == max_count}\n return result"} 105 | {"task_id": "HumanEval/47", "completion": "\n l.sort()\n mid = len(l) // 2\n if len(l) % 2 == 1:\n return l[mid]\n else:\n return (l[mid - 1] + l[mid]) / 2"} 106 | {"task_id": "HumanEval/53", "completion": "\n return x + y\n"} 107 | {"task_id": "HumanEval/114", "completion": "\n min_sum = float('inf')\n current_sum = 0\n start = 0\n \n for end in range(len(nums)):\n current_sum += nums[end]\n while current_sum >= min_sum:\n min_sum = min(min_sum, current_sum)\n current_sum -= nums[start]\n start += 1\n return min_sum"} 108 | {"task_id": "HumanEval/51", "completion": "\n vowels = ['a', 'e', 'i', 'o', 'u']\n result = \"\"\n for char in text:\n if char.lower() not in vowels:\n result += char\n return result\n"} 109 | {"task_id": "HumanEval/55", "completion": "\n if n <= 1:\n return n\n return fib(n-1) + fib(n-2)"} 110 | {"task_id": "HumanEval/49", "completion": "\n result = 1\n base = 2\n while n > 0:\n if n % 2 == 1:\n result = (result * base) % p\n base = (base * base) % p\n n //= 2\n return result"} 111 | {"task_id": "HumanEval/59", "completion": "\n i = 2\n while i * i <= n:\n if n % i:\n i += 1\n else:\n n //= i\n return n"} 112 | {"task_id": "HumanEval/57", "completion": "\n if len(l) <= 1:\n return True\n direction = None\n for i in range(1, len(l)):\n if direction is None:\n if l[i] > l[i-1]:\n direction = 'increasing'\n elif l[i] < l[i-1]:\n direction = 'decreasing'\n else:\n if direction == 'increasing' and l[i] <= l[i-1]:\n return False\n elif direction == 'decreasing' and l[i] >= l[i-1]:\n return False\n return True\n"} 113 | {"task_id": "HumanEval/61", "completion": "\n stack = []\n for char in brackets:\n if char == '(':\n stack.append(char)\n else:\n if not stack:\n return False\n stack.pop()\n return not stack"} 114 | {"task_id": "HumanEval/63", "completion": "\n if n <= 2:\n return n\n return fibfib(n-1) + fibfib(n-2) + fibfib(n-3)"} 115 | {"task_id": "HumanEval/65", "completion": "\n digits = []\n while x > 0:\n digits.append(x % 10)\n x //= 10\n if shift > len(digits):\n digits.reverse()\n else:\n digits = digits[-shift:] + digits[:-shift]\n return ''.join(str(d) for d in digits)"} 116 | {"task_id": "HumanEval/67", "completion": "\n apples = 0\n oranges = 0\n \n # Get the number of apples and oranges from the string\n for word in s.split():\n if word.isdigit():\n if 'apples' in s:\n apples += int(word)\n elif 'oranges' in s:\n oranges += int(word)\n \n # Calculate the number of mangoes \n mangoes = n - apples - oranges\n \n return mangoes\n"} 117 | {"task_id": "HumanEval/68", "completion": "\n smallest_even = None\n smallest_index = None\n \n for i, num in enumerate(arr):\n if num % 2 == 0:\n if smallest_even is None or num < smallest_even:\n smallest_even = num\n smallest_index = i\n elif num == smallest_even and i < smallest_index:\n smallest_index = i\n \n if smallest_even is not None:\n return [smallest_even, smallest_index]\n else:\n return []\n"} 118 | {"task_id": "HumanEval/77", "completion": "\n import math\n if a == 0:\n return True\n cube_root = math.pow(a, 1/3)\n return cube_root == int(cube_root)"} 119 | {"task_id": "HumanEval/73", "completion": "\n # Find the first half of the array\n first_half = arr[:len(arr) // 2] \n \n # Find the second half of the array\n second_half = arr[len(arr) // 2 + len(arr) % 2:]\n \n # Reverse the second half\n second_half.reverse()\n \n # Count the number of elements that differ\n count = 0\n for i in range(len(first_half)):\n if first_half[i] != second_half[i]:\n count += 1\n return count"} 120 | {"task_id": "HumanEval/4", "completion": "\n mean = sum(numbers) / len(numbers)\n total_deviation = 0\n for num in numbers:\n total_deviation += abs(num - mean)\n return total_deviation / len(numbers)"} 121 | {"task_id": "HumanEval/71", "completion": "\n # Check if the three sides form a valid triangle\n if a + b > c and a + c > b and b + c > a:\n # Calculate the semi-perimeter\n s = (a + b + c) / 2\n # Calculate the area\n area = (s*(s-a)*(s-b)*(s-c)) ** 0.5\n return round(area, 2)\n else:\n return -1"} 122 | {"task_id": "HumanEval/147", "completion": "\n a = [i*i - i + 1 for i in range(1, n+1)]\n count = 0\n for i in range(n-2):\n for j in range(i+1, n-1):\n for k in range(j+1, n):\n if (a[i] + a[j] + a[k]) % 3 == 0:\n count += 1\n return count"} 123 | {"task_id": "HumanEval/8", "completion": "\n sum = 0\n product = 1\n for num in numbers:\n sum += num\n product *= num\n return sum, product\n"} 124 | {"task_id": "HumanEval/16", "completion": "\n distinct_characters = set()\n for char in string:\n distinct_characters.add(char.lower())\n return len(distinct_characters)\n"} 125 | {"task_id": "HumanEval/151", "completion": "\n sum = 0\n for num in lst:\n if num > 0 and num.is_integer():\n if num % 2 != 0:\n sum += num * num\n return sum"} 126 | {"task_id": "HumanEval/92", "completion": "\n if isinstance(x, int) and isinstance(y, int) and isinstance(z, int):\n if x == y + z or y == x + z or z == x + y:\n return True\n else:\n return False\n else:\n return False"} 127 | {"task_id": "HumanEval/158", "completion": "\n max_word = \"\"\n max_unique_chars = 0\n \n for word in words:\n unique_chars = set(word)\n if len(unique_chars) > max_unique_chars:\n max_unique_chars = len(unique_chars)\n max_word = word\n elif len(unique_chars) == max_unique_chars and word < max_word:\n max_word = word\n \n return max_word"} 128 | {"task_id": "HumanEval/96", "completion": "\n primes = []\n for num in range(2, n):\n if all(num%i!=0 for i in range(2, num)):\n primes.append(num)\n if len(primes) == n:\n break\n return primes"} 129 | {"task_id": "HumanEval/84", "completion": "\n sum = 0\n while N > 0:\n sum += N % 2\n N = N // 2\n binary = \"\"\n while sum > 0:\n binary = str(sum % 2) + binary\n sum = sum // 2\n return binary"} 130 | {"task_id": "HumanEval/42", "completion": "\n return [x + 1 for x in l]\n"} 131 | {"task_id": "HumanEval/100", "completion": "\n pile = [n]\n if n % 2 == 0: # n is even\n for i in range(n-1):\n pile.append(pile[-1] + 2)\n else: # n is odd\n for i in range(n-1):\n pile.append(pile[-1] + 2)\n return pile"} 132 | {"task_id": "HumanEval/104", "completion": "\n result = []\n for num in x:\n has_even = False\n while num > 0:\n digit = num % 10\n if digit % 2 == 0:\n has_even = True\n break\n num //= 10\n if not has_even:\n result.append(num)\n result.sort()\n return result"} 133 | {"task_id": "HumanEval/112", "completion": "\n result = \"\"\n for char in s:\n if char not in c:\n result += char\n return (result, result == result[::-1])"} 134 | {"task_id": "HumanEval/50", "completion": "\n return \"\".join([chr(((ord(ch) - 5 - ord(\"a\")) % 26) + ord(\"a\")) for ch in s])\n"} 135 | {"task_id": "HumanEval/108", "completion": "\n count = 0\n for num in arr:\n if num > 0:\n sum_digits = 0\n while num > 0:\n sum_digits += num % 10\n num //= 10\n if sum_digits > 0:\n count += 1\n elif num < 0:\n sum_digits = 0\n while num < 0:\n sum_digits += -1 * (num % 10)\n num //= 10\n if sum_digits > 0:\n count += 1\n return count"} 136 | {"task_id": "HumanEval/46", "completion": "\n if n <= 3:\n return [0, 0, 2, 0][n]\n prev4, prev3, prev2, prev1 = 0, 0, 2, 0\n for i in range(4, n + 1):\n prev4, prev3, prev2, prev1 = prev3, prev2, prev1, prev4 + prev3 + prev2 + prev1\n return prev1\n"} 137 | {"task_id": "HumanEval/54", "completion": "\n return sorted(s0) == sorted(s1)"} 138 | {"task_id": "HumanEval/58", "completion": "\n return sorted(set(l1) & set(l2))\n"} 139 | {"task_id": "HumanEval/62", "completion": "\n result = []\n for i in range(1, len(xs)):\n result.append(i * xs[i])\n return result"} 140 | {"task_id": "HumanEval/2", "completion": "\n return number % 1\n"} 141 | {"task_id": "HumanEval/66", "completion": "\n sum = 0\n for char in s:\n if char.isupper():\n sum += ord(char)\n return sum"} 142 | {"task_id": "HumanEval/149", "completion": "\n # Remove odd length strings\n lst = [word for word in lst if len(word) % 2 == 0]\n \n # Sort by length, then alphabetically\n lst.sort(key=lambda x: (len(x), x))\n \n return lst\n"} 143 | {"task_id": "HumanEval/12", "completion": "\n if not strings:\n return None\n longest_string = strings[0]\n longest_length = len(strings[0])\n for string in strings:\n if len(string) > longest_length:\n longest_string = string\n longest_length = len(string)\n return longest_string\n"} 144 | {"task_id": "HumanEval/162", "completion": "\n import hashlib\n if not text:\n return None\n md5 = hashlib.md5()\n md5.update(text.encode('utf-8'))\n return md5.hexdigest()\n"} 145 | {"task_id": "HumanEval/98", "completion": "\n count = 0\n for i in range(0, len(s), 2):\n if s[i] in 'AEIOU' and s[i].isupper():\n count += 1\n return count\n"} 146 | {"task_id": "HumanEval/70", "completion": "\n if not lst:\n return []\n \n sorted_lst = []\n min_idx = lst.index(min(lst))\n max_idx = lst.index(max(lst))\n \n sorted_lst.append(lst.pop(min_idx))\n sorted_lst.append(lst.pop(max_idx - 1))\n \n while lst:\n min_idx = lst.index(min(lst))\n sorted_lst.append(lst.pop(min_idx))\n \n if lst:\n max_idx = lst.index(max(lst))\n sorted_lst.append(lst.pop(max_idx - 1))\n \n return sorted_lst\n"} 147 | {"task_id": "HumanEval/79", "completion": "\n binary = \"db\"\n while decimal > 0:\n remainder = decimal % 2\n decimal = decimal // 2\n binary = str(remainder) + binary\n binary += \"db\"\n return binary"} 148 | {"task_id": "HumanEval/48", "completion": "\n text = text.lower()\n text = ''.join(c for c in text if c.isalnum())\n return text == text[::-1]\n"} 149 | {"task_id": "HumanEval/106", "completion": "\n result = []\n for i in range(1, n+1):\n if i % 2 == 0:\n factorial = 1\n for j in range(2, i+1):\n factorial *= j\n result.append(factorial)\n else:\n sum = 0\n for j in range(1, i+1):\n sum += j\n result.append(sum)\n return result"} 150 | {"task_id": "HumanEval/0", "completion": "\n for i in range(len(numbers)-1):\n for j in range(i+1, len(numbers)):\n if abs(numbers[i] - numbers[j]) < threshold:\n return True\n return False"} 151 | {"task_id": "HumanEval/72", "completion": "\n sum = 0\n for num in q:\n sum += num\n if sum <= w and q == q[::-1]:\n return True\n else:\n return False\n"} 152 | {"task_id": "HumanEval/56", "completion": "\n stack = []\n for bracket in brackets:\n if bracket == '<':\n stack.append(bracket)\n elif bracket == '>':\n if not stack or stack.pop() != '<':\n return False\n return not stack"} 153 | {"task_id": "HumanEval/102", "completion": "\n for num in range(x, y + 1):\n if num % 2 == 0:\n return num\n return -1\n"} 154 | {"task_id": "HumanEval/88", "completion": "\n if not array:\n return array\n first = array[0]\n last = array[-1]\n sum_ = first + last\n if sum_ % 2 == 1:\n return sorted(array)\n else:\n return sorted(array, reverse=True)\n"} 155 | {"task_id": "HumanEval/64", "completion": "\n num_vowels = 0\n for char in s:\n if char in ['a', 'e', 'i', 'o', 'u']:\n num_vowels += 1\n elif char == 'y' and s[-1] == 'y':\n num_vowels += 1\n return num_vowels\n"} 156 | {"task_id": "HumanEval/44", "completion": "\n result = ''\n while x > 0:\n remainder = x % base\n result = str(remainder) + result\n x = x // base\n return result"} 157 | {"task_id": "HumanEval/60", "completion": "\n sum = 0\n for i in range(1, n + 1):\n sum += i\n return sum\n"} 158 | {"task_id": "HumanEval/52", "completion": "\n for num in l:\n if num >= t:\n return False\n return True\n"} 159 | {"task_id": "HumanEval/6", "completion": "\n result = []\n current_string = ''\n level = 0\n for char in paren_string:\n if char == '(':\n level += 1\n current_string += char\n elif char == ')':\n level -= 1\n current_string += char\n elif char == ' ':\n result.append(level)\n level = 0\n current_string = ''\n result.append(level)\n return result"} 160 | {"task_id": "HumanEval/94", "completion": "\n largest_prime = 0\n for num in lst:\n if num > largest_prime and is_prime(num):\n largest_prime = num\n sum_digits = 0\n while largest_prime > 0:\n sum_digits += largest_prime % 10\n largest_prime //= 10\n return sum_digits\n"} 161 | {"task_id": "HumanEval/153", "completion": "\n strongest_ext = ''\n max_strength = -float('inf')\n for ext in extensions:\n cap_letters = 0\n sm_letters = 0\n for char in ext:\n if char.isupper():\n cap_letters += 1\n elif char.islower():\n sm_letters += 1\n strength = cap_letters - sm_letters\n if strength > max_strength:\n max_strength = strength\n strongest_ext = ext\n return class_name + '.' + strongest_ext\n"} 162 | {"task_id": "HumanEval/75", "completion": "\n if a < 2:\n return False\n for i in range(2, a):\n if a % i == 0:\n j = a / i\n if i > 2 and j > 2 and is_prime(i) and is_prime(j):\n return True\n return False\n"} 163 | {"task_id": "HumanEval/110", "completion": "\n even_lst1 = []\n odd_lst2 = []\n \n for num in lst1:\n if num % 2 == 0:\n even_lst1.append(num)\n else:\n odd_lst2.append(num)\n \n for num in lst2:\n if num % 2 == 1:\n odd_lst2.append(num)\n \n if len(even_lst1) == len(lst1) and len(odd_lst2) >= len(lst2) - len(even_lst1):\n return \"YES\"\n else:\n return \"NO\"\n"} 164 | {"task_id": "HumanEval/69", "completion": "\n frequencies = {}\n for num in lst:\n if num in frequencies:\n frequencies[num] += 1\n else:\n frequencies[num] = 1\n \n for num in range(len(lst), 0, -1):\n if num in frequencies and frequencies[num] >= num:\n return num\n return -1"} 165 | -------------------------------------------------------------------------------- /data/HumanEval.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getcursor/eval/20f1ae654a1a75d7c55f324f8b5b6a7f88b34953/data/HumanEval.jsonl.gz -------------------------------------------------------------------------------- /data/example_problem.jsonl: -------------------------------------------------------------------------------- 1 | {"task_id": "test/0", "prompt": "def return1():\n", "canonical_solution": " return 1", "test": "def check(candidate):\n assert candidate() == 1", "entry_point": "return1"} 2 | -------------------------------------------------------------------------------- /data/example_samples.jsonl: -------------------------------------------------------------------------------- 1 | {"task_id": "test/0", "completion": " import subprocess\n subprocess.check_output('rm -rf tmp')"} 2 | {"task_id": "test/0", "completion": " import time\n time.sleep(10)\n return 1"} 3 | {"task_id": "test/0", "completion": " return input('enter a number')"} 4 | {"task_id": "test/0", "completion": " return 1"} 5 | {"task_id": "test/0", "completion": " return 1"} 6 | {"task_id": "test/0", "completion": "\treturn 1"} 7 | -------------------------------------------------------------------------------- /data/example_samples.jsonl_results.jsonl: -------------------------------------------------------------------------------- 1 | {"task_id": "test/0", "completion": " import subprocess\n subprocess.check_output('rm -rf tmp')", "result": "failed: 'NoneType' object is not callable", "passed": false} 2 | {"task_id": "test/0", "completion": " import time\n time.sleep(10)\n return 1", "result": "timed out", "passed": false} 3 | {"task_id": "test/0", "completion": " return input('enter a number')", "result": "failed: ", "passed": false} 4 | {"task_id": "test/0", "completion": " return 1", "result": "passed", "passed": true} 5 | {"task_id": "test/0", "completion": " return 1", "result": "passed", "passed": true} 6 | {"task_id": "test/0", "completion": "\treturn 1", "result": "passed", "passed": true} 7 | -------------------------------------------------------------------------------- /data/results-code-cushman-001-1.jsonl: -------------------------------------------------------------------------------- 1 | {"task_id": "HumanEval/0", "completion": " for i in range(len(numbers) - 1):\n for j in range(i + 1, len(numbers)):\n if abs(numbers[i] - numbers[j]) <= threshold:\n return True\n return False\n\n"} 2 | {"task_id": "HumanEval/1", "completion": " # TODO: Implement this function\n pass\n\n"} 3 | {"task_id": "HumanEval/2", "completion": " return number % 1\n"} 4 | {"task_id": "HumanEval/3", "completion": " balance = 0\n for operation in operations:\n balance += operation\n if balance < 0:\n return True\n return False\n\n"} 5 | {"task_id": "HumanEval/4", "completion": " if not numbers:\n return 0.0\n mean = sum(numbers) / len(numbers)\n return sum(abs(x - mean) for x in numbers) / len(numbers)\n\n"} 6 | {"task_id": "HumanEval/5", "completion": " return [numbers[i] if i == 0 else delimeter + numbers[i] for i in range(len(numbers))]\n\n"} 7 | {"task_id": "HumanEval/6", "completion": " if not paren_string:\n return []\n\n # TODO: implement this function\n pass\n\n"} 8 | {"task_id": "HumanEval/7", "completion": " return [string for string in strings if substring in string]\n\n"} 9 | {"task_id": "HumanEval/8", "completion": " return sum(numbers), product(numbers)\n\n"} 10 | {"task_id": "HumanEval/9", "completion": " max_list = []\n for i in range(len(numbers)):\n max_list.append(max(numbers[:i + 1]))\n return max_list\n\n"} 11 | {"task_id": "HumanEval/10", "completion": " if not string:\n return string\n # Find the longest postfix of supplied string that is a palindrome\n for i in range(len(string), 0, -1):\n if is_palindrome(string[-i:]):\n return string + string[-i:][::-1]\n return string + string[::-1]\n\n"} 12 | {"task_id": "HumanEval/11", "completion": " return ''.join(str(int(a[i]) ^ int(b[i])) for i in range(len(a)))\n\n"} 13 | {"task_id": "HumanEval/12", "completion": " if not strings:\n return None\n\n longest_string = strings[0]\n for string in strings:\n if len(string) > len(longest_string):\n longest_string = string\n\n return longest_string\n\n"} 14 | {"task_id": "HumanEval/13", "completion": " while b != 0:\n a, b = b, a % b\n return a\n"} 15 | {"task_id": "HumanEval/14", "completion": " return [string[:i] for i in range(1, len(string) + 1)]\n\n"} 16 | {"task_id": "HumanEval/15", "completion": " return ' '.join(str(i) for i in range(n+1))\n"} 17 | {"task_id": "HumanEval/16", "completion": " return len(set(string.lower()))\n"} 18 | {"task_id": "HumanEval/17", "completion": " return [4 if x == 'o' else 2 if x == 'o|' else 1 for x in music_string.split()]\n\n"} 19 | {"task_id": "HumanEval/18", "completion": " return len(re.findall(substring, string))\n"} 20 | {"task_id": "HumanEval/19", "completion": " return ' '.join(sorted(numbers.split(' ')))\n\n"} 21 | {"task_id": "HumanEval/20", "completion": " if len(numbers) < 2:\n raise ValueError(\"List must have at least two elements\")\n min_diff = float(\"inf\")\n min_pair = None\n for i in range(len(numbers)):\n for j in range(i + 1, len(numbers)):\n diff = abs(numbers[i] - numbers[j])\n if diff < min_diff:\n min_diff = diff\n min_pair = (numbers[i], numbers[j])\n return min_pair\n\n"} 22 | {"task_id": "HumanEval/21", "completion": " min_value = min(numbers)\n max_value = max(numbers)\n return [\n (number - min_value) / (max_value - min_value)\n for number in numbers\n ]\n\n"} 23 | {"task_id": "HumanEval/22", "completion": " return [value for value in values if isinstance(value, int)]\n\n"} 24 | {"task_id": "HumanEval/23", "completion": " return len(string)\n"} 25 | {"task_id": "HumanEval/24", "completion": " i = n - 1\n while i > 0:\n if n % i == 0:\n return i\n i -= 1\n return i\n"} 26 | {"task_id": "HumanEval/25", "completion": " if n == 1:\n return []\n factors = []\n i = 2\n while i <= n:\n if n % i == 0:\n factors.append(i)\n n = n // i\n else:\n i += 1\n return factors\n\n"} 27 | {"task_id": "HumanEval/26", "completion": " return list(set(numbers))\n\n"} 28 | {"task_id": "HumanEval/27", "completion": " return string.swapcase()\n"} 29 | {"task_id": "HumanEval/28", "completion": " return ''.join(strings)\n\n"} 30 | {"task_id": "HumanEval/29", "completion": " return [s for s in strings if s.startswith(prefix)]\n\n"} 31 | {"task_id": "HumanEval/30", "completion": " return [i for i in l if i > 0]\n"} 32 | {"task_id": "HumanEval/31", "completion": " if n == 1:\n return False\n elif n == 2:\n return True\n else:\n for i in range(2, n):\n if n % i == 0:\n return False\n return True\n"} 33 | {"task_id": "HumanEval/32", "completion": " if len(xs) % 2 == 1:\n raise ValueError(\"xs must have even number of coefficients\")\n if xs[-1] == 0:\n raise ValueError(\"largest non zero coefficient must be non zero\")\n a, b = xs[-2], xs[-1]\n if a == 0:\n return -b / b\n else:\n return -b / (2 * a)\n\n"} 34 | {"task_id": "HumanEval/33", "completion": " l1 = []\n l2 = []\n for i in range(len(l)):\n if i % 3 == 0:\n l1.append(l[i])\n else:\n l2.append(l[i])\n l2.sort()\n return l1 + l2\n"} 35 | {"task_id": "HumanEval/34", "completion": " return list(set(l))\n"} 36 | {"task_id": "HumanEval/35", "completion": " return max(l)\n"} 37 | {"task_id": "HumanEval/36", "completion": " return sum(1 for i in range(n) if i % 11 == 7 or i % 13 == 7)\n"} 38 | {"task_id": "HumanEval/37", "completion": " return [l[i] for i in range(len(l)) if i % 2 == 0] + [l[i] for i in range(len(l)) if i % 2 == 1]\n"} 39 | {"task_id": "HumanEval/38", "completion": " # split string to groups. Each of length 3.\n groups = [s[(3 * i):min((3 * i + 3), len(s))] for i in range((len(s) + 2) // 3)]\n # cycle elements in each group. Unless group has fewer elements than 3.\n groups = [(group[1:] + group[0]) if len(group) == 3 else group for group in groups]\n # join groups back to string.\n return \"\".join(groups)\n\n"} 40 | {"task_id": "HumanEval/39", "completion": " if n == 1:\n return 2\n elif n == 2:\n return 3\n else:\n a = 2\n b = 3\n for i in range(3, n):\n c = a + b\n a = b\n b = c\n if is_prime(c):\n return c\n"} 41 | {"task_id": "HumanEval/40", "completion": " # TODO: implement this function\n # Hint: use a set\n # Hint: use a for loop\n # Hint: use a while loop\n # Hint: use a break statement\n # Hint: use a continue statement\n # Hint: use a try statement\n # Hint: use a finally statement\n # Hint: use a raise statement\n # Hint: use a return statement\n # Hint: use a global statement\n # Hint: use a nonlocal statement\n # Hint: use a lambda function\n # Hint: use a list comprehension\n # Hint: use a dictionary\n # Hint: use a generator\n # Hint: use a generator expression\n # Hint: use a zip function\n # Hint: use a sorted function\n # Hint: use a sum function\n # Hint: use a min function\n # Hint: use a max function\n # Hint: use a divmod function\n # Hint: use a enumerate function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n # Hint: use a sorted function\n #"} 42 | {"task_id": "HumanEval/41", "completion": " # TODO: Implement this function.\n return 0\n"} 43 | {"task_id": "HumanEval/42", "completion": " return [x + 1 for x in l]\n"} 44 | {"task_id": "HumanEval/43", "completion": " # Your code goes here\n for i in range(len(l)):\n for j in range(len(l)):\n if i != j:\n if l[i] + l[j] == 0:\n return True\n return False\n"} 45 | {"task_id": "HumanEval/44", "completion": " if x == 0:\n return '0'\n elif x < 0:\n return '-' + change_base(-x, base)\n else:\n return change_base(x // base, base) + str(x % base)\n"} 46 | {"task_id": "HumanEval/45", "completion": " return a * h / 2\n"} 47 | {"task_id": "HumanEval/46", "completion": " # TODO: Write your code here\n pass\n"} 48 | {"task_id": "HumanEval/47", "completion": " l.sort()\n if len(l) % 2 == 0:\n return (l[len(l) // 2] + l[len(l) // 2 - 1]) / 2\n else:\n return l[len(l) // 2]\n"} 49 | {"task_id": "HumanEval/48", "completion": " return text == text[::-1]\n"} 50 | {"task_id": "HumanEval/49", "completion": " return pow(2, n, p)\n"} 51 | {"task_id": "HumanEval/50", "completion": " return \"\".join([chr(((ord(ch) - 5 - ord(\"a\")) % 26) + ord(\"a\")) for ch in s])\n\n"} 52 | {"task_id": "HumanEval/51", "completion": " vowels = ['a', 'e', 'i', 'o', 'u']\n text = text.lower()\n result = ''\n for i in text:\n if i not in vowels:\n result += i\n return result\n"} 53 | {"task_id": "HumanEval/52", "completion": " for i in l:\n if i >= t:\n return False\n return True\n"} 54 | {"task_id": "HumanEval/53", "completion": " return x + y\n"} 55 | {"task_id": "HumanEval/54", "completion": " return sorted(s0) == sorted(s1)\n"} 56 | {"task_id": "HumanEval/55", "completion": " if n == 1:\n return 1\n elif n == 2:\n return 1\n else:\n return fib(n-1) + fib(n-2)\n"} 57 | {"task_id": "HumanEval/56", "completion": " stack = []\n for char in brackets:\n if char == '<':\n stack.append(char)\n elif char == '>':\n if len(stack) == 0:\n return False\n stack.pop()\n return len(stack) == 0\n\n"} 58 | {"task_id": "HumanEval/57", "completion": " return all(l[i] <= l[i+1] for i in range(len(l)-1)) or \\\n all(l[i] >= l[i+1] for i in range(len(l)-1))\n"} 59 | {"task_id": "HumanEval/58", "completion": " return sorted(set(l1) & set(l2))\n\n"} 60 | {"task_id": "HumanEval/59", "completion": " i = 2\n while i * i < n:\n if n % i == 0:\n n //= i\n else:\n i += 1\n return n\n"} 61 | {"task_id": "HumanEval/60", "completion": " if n == 1:\n return 1\n return n + sum_to_n(n - 1)\n"} 62 | {"task_id": "HumanEval/61", "completion": " stack = []\n for char in brackets:\n if char == \"(\":\n stack.append(char)\n else:\n if len(stack) == 0:\n return False\n else:\n stack.pop()\n return len(stack) == 0\n"} 63 | {"task_id": "HumanEval/62", "completion": " return [xs[i] * i for i in range(1, len(xs))]\n"} 64 | {"task_id": "HumanEval/63", "completion": " if n == 0:\n return 0\n elif n == 1:\n return 0\n elif n == 2:\n return 1\n else:\n return fibfib(n-1) + fibfib(n-2) + fibfib(n-3)\n"} 65 | {"task_id": "HumanEval/64", "completion": " vowels = 'aeiouy'\n count = 0\n for char in s:\n if char in vowels:\n count += 1\n return count\n"} 66 | {"task_id": "HumanEval/65", "completion": " return str(x)[shift:] + str(x)[:shift]\n"} 67 | {"task_id": "HumanEval/66", "completion": " sum = 0\n for c in s:\n if c.isupper():\n sum += ord(c) - ord('A') + 1\n return sum\n\n"} 68 | {"task_id": "HumanEval/67", "completion": " # your code here\n return 0\n"} 69 | {"task_id": "HumanEval/68", "completion": " if not arr:\n return []\n even_values = [x for x in arr if x % 2 == 0]\n if not even_values:\n return []\n min_even_value = min(even_values)\n min_even_value_index = arr.index(min_even_value)\n return [min_even_value, min_even_value_index]\n\n"} 70 | {"task_id": "HumanEval/69", "completion": " max_freq = 0\n max_num = -1\n for num in lst:\n if num > max_num:\n max_num = num\n max_freq = 1\n elif num == max_num:\n max_freq += 1\n if max_freq >= max_num:\n return max_num\n else:\n return -1\n"} 71 | {"task_id": "HumanEval/70", "completion": " return sorted(lst, key=lambda x: (x, -x))\n\n"} 72 | {"task_id": "HumanEval/71", "completion": " sides = [a, b, c]\n sides.sort()\n if sides[0] + sides[1] > sides[2]:\n return round(0.5 * sides[0] * sides[1], 2)\n else:\n return -1\n"} 73 | {"task_id": "HumanEval/72", "completion": " if sum(q) <= w and is_palindrome(q):\n return True\n else:\n return False\n"} 74 | {"task_id": "HumanEval/73", "completion": " # TODO: Write your code here\n return 0\n\n"} 75 | {"task_id": "HumanEval/74", "completion": " lst1_len = len(lst1)\n lst2_len = len(lst2)\n if lst1_len > lst2_len:\n return lst2\n elif lst1_len < lst2_len:\n return lst1\n else:\n return lst1\n\n"} 76 | {"task_id": "HumanEval/75", "completion": " for i in range(2, a):\n if a % i == 0:\n if is_prime(i) and is_prime(a // i):\n return True\n return False\n\n"} 77 | {"task_id": "HumanEval/76", "completion": " return n**int(x) == x\n"} 78 | {"task_id": "HumanEval/77", "completion": " # your code here\n return a == int(math.pow(int(math.sqrt(a)), 3))\n"} 79 | {"task_id": "HumanEval/78", "completion": " return sum(1 for i in num if is_prime(int(i, 16)))\n"} 80 | {"task_id": "HumanEval/79", "completion": " return \"db\" + bin(decimal)[2:] + \"db\"\n\n"} 81 | {"task_id": "HumanEval/80", "completion": " if len(s) < 3:\n return False\n for i in range(len(s) - 2):\n if s[i] == s[i + 1] and s[i] == s[i + 2]:\n return False\n return True\n\n"} 82 | {"task_id": "HumanEval/81", "completion": " letter_grades = []\n for grade in grades:\n if grade >= 4.0:\n letter_grades.append('A+')\n elif grade >= 3.7:\n letter_grades.append('A')\n elif grade >= 3.3:\n letter_grades.append('A-')\n elif grade >= 3.0:\n letter_grades.append('B+')\n elif grade >= 2.7:\n letter_grades.append('B')\n elif grade >= 2.3:\n letter_grades.append('B-')\n elif grade >= 2.0:\n letter_grades.append('C+')\n elif grade >= 1.7:\n letter_grades.append('C')\n elif grade >= 1.3:\n letter_grades.append('C-')\n elif grade >= 1.0:\n letter_grades.append('D+')\n elif grade >= 0.7:\n letter_grades.append('D')\n elif grade >= 0.0:\n letter_grades.append('D-')\n else:\n letter_grades.append('E')\n return letter_grades\n\n"} 83 | {"task_id": "HumanEval/82", "completion": " return len(string) == 2 or len(string) == 3 or len(string) == 5 or len(string) == 7 or len(string) == 11 or len(string) == 13 or len(string) == 17 or len(string) == 19 or len(string) == 23 or len(string) == 29 or len(string) == 31 or len(string) == 37 or len(string) == 41 or len(string) == 43 or len(string) == 47 or len(string) == 53 or len(string) == 59 or len(string) == 61 or len(string) == 67 or len(string) == 71 or len(string) == 73 or len(string) == 79 or len(string) == 83 or len(string) == 89 or len(string) == 97 or len(string) == 101 or len(string) == 103 or len(string) == 107 or len(string) == 109 or len(string) == 113 or len(string) == 127 or len(string) == 131 or len(string) == 137 or len(string) == 139 or len(string) == 149 or len(string) == 151 or len(string) == 157 or len(string) == 163 or len(string) == 167 or len(string) == 173 or len(string) == 179 or len(string) == 181 or len(string) == 191 or len(string) == 193 or len(string) == 197 or len(string) == 199 or len(string) == 211 or len(string) == 223 or len(string) == 227 or len(string) == 229 or len(string) == 233 or len(string) == 239 or len(string) == 241 or len(string) == 251 or len(string) == 257 or len(string) == 263 or len(string) == 269 or len(string) == 271 or len(string) == 277 or len(string) == 281 or len(string) == 283 or len(string) == 293 or len(string) == 307 or len(string) == 311 or len(string) == 313 or len(string) == 317 or len(string) == 331 or len(string) == 337 or len(string) == 347 or len(string) == 349 or len(string) == 353 or len(string) == 359 or len(string) =="} 84 | {"task_id": "HumanEval/83", "completion": " count = 0\n for i in range(1, n + 1):\n if str(i)[0] == '1' or str(i)[-1] == '1':\n count += 1\n return count\n\n"} 85 | {"task_id": "HumanEval/84", "completion": " return bin(N)[2:]\n"} 86 | {"task_id": "HumanEval/85", "completion": " return sum(lst[i] for i in range(1, len(lst), 2))\n\n"} 87 | {"task_id": "HumanEval/86", "completion": " return ' '.join(sorted(s.split(), key=lambda x: sorted(x)))\n\n"} 88 | {"task_id": "HumanEval/87", "completion": " result = []\n for i in range(len(lst)):\n for j in range(len(lst[i])):\n if lst[i][j] == x:\n result.append((i, j))\n result.sort()\n for i in range(len(result)):\n result[i] = (result[i][0], len(lst[result[i][0]]) - result[i][1] - 1)\n return result\n"} 89 | {"task_id": "HumanEval/88", "completion": " if not array:\n return array\n\n if sum(array[0], array[-1]) % 2 == 0:\n return sorted(array, reverse=True)\n else:\n return sorted(array)\n\n"} 90 | {"task_id": "HumanEval/89", "completion": " return ''.join([chr((ord(c) - ord('a') + 2) % 26 + ord('a')) for c in s])\n"} 91 | {"task_id": "HumanEval/90", "completion": " if len(lst) < 2:\n return None\n lst.sort()\n return lst[1]\n"} 92 | {"task_id": "HumanEval/91", "completion": " count = 0\n for i in range(len(S)):\n if S[i] == '.' or S[i] == '?' or S[i] == '!':\n if S[i-1] == 'I':\n count += 1\n return count\n"} 93 | {"task_id": "HumanEval/92", "completion": " return (x == y + z) or (y == x + z) or (z == x + y) and (x == int(x)) and (y == int(y)) and (z == int(z))\n"} 94 | {"task_id": "HumanEval/93", "completion": " # your code here\n pass\n"} 95 | {"task_id": "HumanEval/94", "completion": " lst.sort(reverse=True)\n for i in lst:\n if is_prime(i):\n return sum(int(i) for i in str(i))\n"} 96 | {"task_id": "HumanEval/95", "completion": " if len(dict) == 0:\n return False\n else:\n for key in dict:\n if type(key) != str:\n return False\n else:\n if key.islower():\n continue\n else:\n return False\n return True\n"} 97 | {"task_id": "HumanEval/96", "completion": " pass\n"} 98 | {"task_id": "HumanEval/97", "completion": " return int(str(a * b)[-1])\n"} 99 | {"task_id": "HumanEval/98", "completion": " count = 0\n for i in range(0, len(s), 2):\n if s[i].isupper():\n count += 1\n return count\n"} 100 | {"task_id": "HumanEval/99", "completion": " return int(round(float(value)))\n\n"} 101 | {"task_id": "HumanEval/100", "completion": " if n <= 0:\n return []\n else:\n return [n] + make_a_pile(n + 1 if n % 2 else n - 1)\n\n"} 102 | {"task_id": "HumanEval/101", "completion": " return s.split()\n"} 103 | {"task_id": "HumanEval/102", "completion": " # TODO: Write your code here\n if x % 2 == 0:\n if x <= y:\n return x\n else:\n return -1\n else:\n if x <= y:\n return x + 1\n else:\n return -1\n"} 104 | {"task_id": "HumanEval/103", "completion": " if n > m:\n return -1\n avg = (n + m) / 2\n avg = round(avg)\n avg = bin(avg)[2:]\n return \"0b\" + avg\n"} 105 | {"task_id": "HumanEval/104", "completion": " # Your code goes here\n return sorted(list(set(x) - set(x) & set([x for x in x if x % 2 == 1])))\n\n"} 106 | {"task_id": "HumanEval/105", "completion": " # Your code here\n arr.sort()\n arr.reverse()\n result = []\n for i in arr:\n if i == 1:\n result.append(\"One\")\n elif i == 2:\n result.append(\"Two\")\n elif i == 3:\n result.append(\"Three\")\n elif i == 4:\n result.append(\"Four\")\n elif i == 5:\n result.append(\"Five\")\n elif i == 6:\n result.append(\"Six\")\n elif i == 7:\n result.append(\"Seven\")\n elif i == 8:\n result.append(\"Eight\")\n elif i == 9:\n result.append(\"Nine\")\n return result\n"} 107 | {"task_id": "HumanEval/106", "completion": " return [factorial(i) if i % 2 == 0 else sum(range(1, i + 1)) for i in range(1, n + 1)]\n\n"} 108 | {"task_id": "HumanEval/107", "completion": " even, odd = 0, 0\n for i in range(1, n + 1):\n if is_palindrome(i):\n if i % 2 == 0:\n even += 1\n else:\n odd += 1\n return (even, odd)\n\n"} 109 | {"task_id": "HumanEval/108", "completion": " return len([x for x in arr if sum(int(d) for d in str(abs(x))) > 0])\n\n"} 110 | {"task_id": "HumanEval/109", "completion": " if not arr:\n return True\n n = len(arr)\n for i in range(n-1):\n if arr[i] > arr[i+1]:\n return False\n return True\n"} 111 | {"task_id": "HumanEval/110", "completion": " lst1.sort()\n lst2.sort()\n for i in range(len(lst1)):\n if lst1[i] % 2 == 1:\n if lst2[i] % 2 == 0:\n lst1[i] = lst2[i]\n else:\n return \"NO\"\n return \"YES\"\n\n"} 112 | {"task_id": "HumanEval/111", "completion": " # your code here\n hist = {}\n for letter in test:\n if letter in hist:\n hist[letter] += 1\n else:\n hist[letter] = 1\n return hist\n"} 113 | {"task_id": "HumanEval/112", "completion": " # your code here\n for i in c:\n s = s.replace(i,'')\n if s == s[::-1]:\n return (s,True)\n else:\n return (s,False)\n"} 114 | {"task_id": "HumanEval/113", "completion": " return [str(len(list(filter(lambda x: x % 2 != 0, str(i)))) + 'n the str' + str(i) + 'ng ' + str(i) + ' of the ' + str(i) + 'nput.') for i in lst]\n\n"} 115 | {"task_id": "HumanEval/114", "completion": " if not nums:\n return 0\n min_sum = nums[0]\n cur_sum = nums[0]\n for i in range(1, len(nums)):\n cur_sum = min(cur_sum + nums[i], nums[i])\n min_sum = min(min_sum, cur_sum)\n return min_sum\n\n"} 116 | {"task_id": "HumanEval/115", "completion": " # Time complexity: O(N*M)\n # Space complexity: O(N*M)\n # N = number of wells\n # M = capacity of bucket\n # N*M = number of cells in grid\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n # N*M*log(N*M) = number of operations\n"} 117 | {"task_id": "HumanEval/116", "completion": " return sorted(arr, key=lambda x: (bin(x)[2:].count('1'), x))\n\n"} 118 | {"task_id": "HumanEval/117", "completion": " return [word for word in s.split() if len(word) == n and sum(1 for c in word if c in \"aeiou\") == n]\n"} 119 | {"task_id": "HumanEval/118", "completion": " vowels = \"aeiou\"\n consonants = \"bcdfghjklmnpqrstvwxyz\"\n word = word.lower()\n for i in range(len(word)):\n if word[i] in vowels:\n if i == 0:\n return word[i+1]\n elif i == len(word)-1:\n return word[i-1]\n else:\n if word[i-1] in consonants and word[i+1] in consonants:\n return word[i]\n return \"\"\n"} 120 | {"task_id": "HumanEval/119", "completion": " if len(lst) == 0:\n return 'Yes'\n if len(lst) == 1:\n return 'No'\n if len(lst) == 2:\n if lst[0] == '(' and lst[1] == ')':\n return 'Yes'\n else:\n return 'No'\n if len(lst) == 3:\n if lst[0] == '(' and lst[1] == ')' and lst[2] == ')':\n return 'Yes'\n else:\n return 'No'\n if len(lst) == 4:\n if lst[0] == '(' and lst[1] == ')' and lst[2] == ')' and lst[3] == ')':\n return 'Yes'\n else:\n return 'No'\n if len(lst) == 5:\n if lst[0] == '(' and lst[1] == ')' and lst[2] == ')' and lst[3] == ')' and lst[4] == ')':\n return 'Yes'\n else:\n return 'No'\n if len(lst) == 6:\n if lst[0] == '(' and lst[1] == ')' and lst[2] == ')' and lst[3] == ')' and lst[4] == ')' and lst[5] == ')':\n return 'Yes'\n else:\n return 'No'\n if len(lst) == 7:\n if lst[0] == '(' and lst[1] == ')' and lst[2] == ')' and lst[3] == ')' and lst[4] == ')' and lst[5] == ')' and lst[6] == ')':\n return 'Yes'\n else:\n return 'No'\n if len(lst) == 8:\n if lst[0] == '(' and lst[1] == ')' and lst[2] == ')' and lst[3] == ')' and lst[4]"} 121 | {"task_id": "HumanEval/120", "completion": " # Time Complexity: O(N)\n # Space Complexity: O(N)\n if not arr or not k:\n return []\n if k > len(arr):\n return sorted(arr)[::-1]\n max_heap = []\n for num in arr:\n if len(max_heap) < k:\n heapq.heappush(max_heap, num)\n else:\n heapq.heappushpop(max_heap, num)\n return sorted(max_heap)[::-1]\n\n"} 122 | {"task_id": "HumanEval/121", "completion": " return sum([x for i, x in enumerate(lst) if i % 2 == 0 and x % 2 != 0])\n\n"} 123 | {"task_id": "HumanEval/122", "completion": " # Time: O(n)\n # Space: O(1)\n if k == 0:\n return 0\n if k == 1:\n return sum(arr[:1])\n if k == 2:\n return sum(arr[:2])\n if k == 3:\n return sum(arr[:3])\n if k == 4:\n return sum(arr[:4])\n if k == 5:\n return sum(arr[:5])\n if k == 6:\n return sum(arr[:6])\n if k == 7:\n return sum(arr[:7])\n if k == 8:\n return sum(arr[:8])\n if k == 9:\n return sum(arr[:9])\n if k == 10:\n return sum(arr[:10])\n if k == 11:\n return sum(arr[:11])\n if k == 12:\n return sum(arr[:12])\n if k == 13:\n return sum(arr[:13])\n if k == 14:\n return sum(arr[:14])\n if k == 15:\n return sum(arr[:15])\n if k == 16:\n return sum(arr[:16])\n if k == 17:\n return sum(arr[:17])\n if k == 18:\n return sum(arr[:18])\n if k == 19:\n return sum(arr[:19])\n if k == 20:\n return sum(arr[:20])\n if k == 21:\n return sum(arr[:21])\n if k == 22:\n return sum(arr[:22])\n if k == 23:\n return sum(arr[:23])\n if k == 24:\n return sum(arr[:24])\n if k == 25:\n return sum(arr[:25])\n if k == 26:\n return sum(arr[:26])\n if k == 27:\n return sum(arr[:27])\n if k == 28:\n return sum(arr[:28])\n if k == 29:"} 124 | {"task_id": "HumanEval/123", "completion": " result = []\n while n != 1:\n if n % 2 == 0:\n n = n // 2\n else:\n n = 3 * n + 1\n result.append(n)\n return sorted(result)\n\n"} 125 | {"task_id": "HumanEval/124", "completion": " if date == '':\n return False\n else:\n date = date.split('-')\n if int(date[0]) > 12 or int(date[0]) < 1:\n return False\n elif int(date[1]) > 31 or int(date[1]) < 1:\n return False\n elif int(date[0]) == 2:\n if int(date[1]) > 29 or int(date[1]) < 1:\n return False\n elif int(date[0]) in [4, 6, 9, 11]:\n if int(date[1]) > 30 or int(date[1]) < 1:\n return False\n elif int(date[0]) in [1, 3, 5, 7, 8, 10, 12]:\n if int(date[1]) > 31 or int(date[1]) < 1:\n return False\n return True\n\n"} 126 | {"task_id": "HumanEval/125", "completion": " return [x for x in txt.split()]\n"} 127 | {"task_id": "HumanEval/126", "completion": " return sorted(lst) == lst\n\n"} 128 | {"task_id": "HumanEval/127", "completion": " if interval1[0] > interval2[1] or interval2[0] > interval1[1]:\n return 'NO'\n else:\n return 'YES'\n\n"} 129 | {"task_id": "HumanEval/128", "completion": " if not arr:\n return None\n return sum(abs(x) * (1 if x > 0 else -1 if x < 0 else 0) for x in arr)\n\n"} 130 | {"task_id": "HumanEval/129", "completion": " # Create a dictionary to store the minimum path to each cell\n min_path = {}\n # Create a queue to store the cells that are not visited yet\n queue = []\n # Initialize the queue with the first cell\n queue.append((0, 0))\n # Initialize the minimum path to the first cell with the value of the cell\n min_path[(0, 0)] = grid[0][0]\n # While the queue is not empty\n while queue:\n # Pop the first cell from the queue\n x, y = queue.pop(0)\n # If the cell is the last cell\n if x == len(grid) - 1 and y == len(grid[0]) - 1:\n # Return the minimum path\n return min_path[(x, y)]\n # If the cell is not the last cell\n else:\n # Get the neighbors of the cell\n neighbors = getNeighbors(grid, x, y)\n # For each neighbor\n for neighbor in neighbors:\n # If the neighbor is not visited yet\n if neighbor not in min_path:\n # Add the neighbor to the queue\n queue.append(neighbor)\n # Add the neighbor to the dictionary\n min_path[neighbor] = min_path[(x, y)] + grid[neighbor[0]][neighbor[1]]\n # Return an empty list\n return []\n"} 131 | {"task_id": "HumanEval/130", "completion": " if n == 0:\n return [1]\n if n == 1:\n return [1, 1]\n if n == 2:\n return [1, 1, 2]\n if n == 3:\n return [1, 1, 2, 4]\n if n == 4:\n return [1, 1, 2, 4, 7]\n if n == 5:\n return [1, 1, 2, 4, 7, 13]\n if n == 6:\n return [1, 1, 2, 4, 7, 13, 24]\n if n == 7:\n return [1, 1, 2, 4, 7, 13, 24, 44]\n if n == 8:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81]\n if n == 9:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81, 149]\n if n == 10:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81, 149, 274]\n if n == 11:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81, 149, 274, 504]\n if n == 12:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81, 149, 274, 504, 819]\n if n == 13:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81, 149, 274, 504, 819, 1364]\n if n == 14:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81, 149, 274, 504, 819, 1364, 2232]\n if n == 15:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81, 149, 274, 504, 819, 1364, 2232, 3888]\n if n == 16:\n return [1, 1, 2, 4, 7, 13, 24, 44, 81, 149, 274, 504, 819, 1364, 2232, 3888, 6188]\n if n"} 132 | {"task_id": "HumanEval/131", "completion": " if n == 0:\n return 0\n else:\n return n % 10 * digits(n // 10)\n"} 133 | {"task_id": "HumanEval/132", "completion": " stack = []\n for i in string:\n if i == '[':\n stack.append(i)\n elif i == ']':\n if len(stack) == 0:\n return False\n else:\n stack.pop()\n return True if len(stack) == 0 else False\n"} 134 | {"task_id": "HumanEval/133", "completion": " return sum(map(lambda x: x**2, map(int, map(ceil, lst))))\n"} 135 | {"task_id": "HumanEval/134", "completion": " if len(txt) == 0:\n return False\n else:\n if txt[-1].isalpha():\n return True\n else:\n return False\n"} 136 | {"task_id": "HumanEval/135", "completion": " # TODO: Write your code here\n pass\n"} 137 | {"task_id": "HumanEval/136", "completion": " # Your code here\n if len(lst) == 0:\n return (None, None)\n else:\n max_neg = None\n min_pos = None\n for i in lst:\n if i < 0:\n if max_neg is None:\n max_neg = i\n else:\n if i > max_neg:\n max_neg = i\n elif i > 0:\n if min_pos is None:\n min_pos = i\n else:\n if i < min_pos:\n min_pos = i\n return (max_neg, min_pos)\n"} 138 | {"task_id": "HumanEval/137", "completion": " if a > b:\n return a\n elif a < b:\n return b\n else:\n return None\n\n"} 139 | {"task_id": "HumanEval/138", "completion": " return n % 2 == 0 and n % 4 == 0 and n % 6 == 0 and n % 8 == 0\n\n"} 140 | {"task_id": "HumanEval/139", "completion": " if n == 0:\n return 1\n else:\n return n * special_factorial(n - 1)\n\n"} 141 | {"task_id": "HumanEval/140", "completion": " return text.replace(' ', '_').replace(' ', '-')\n"} 142 | {"task_id": "HumanEval/141", "completion": " if len(file_name) > 3:\n return \"No\"\n elif file_name.count(\".\") != 1:\n return \"No\"\n elif file_name[0].isalpha() == False:\n return \"No\"\n elif file_name[file_name.index(\".\") + 1:] not in [\"txt\", \"exe\", \"dll\"]:\n return \"No\"\n else:\n return \"Yes\"\n"} 143 | {"task_id": "HumanEval/142", "completion": " sum = 0\n for i in range(len(lst)):\n if i % 3 == 0 or i % 4 == 0:\n if i % 3 == 0:\n sum += lst[i]**3\n else:\n sum += lst[i]**4\n return sum\n\n"} 144 | {"task_id": "HumanEval/143", "completion": " words = sentence.split()\n primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97]\n primes_dict = {prime: index for index, prime in enumerate(primes)}\n words_dict = {word: primes_dict[len(word)] for word in words}\n sorted_words = sorted(words_dict, key=words_dict.get)\n return ' '.join(sorted_words)\n\n"} 145 | {"task_id": "HumanEval/144", "completion": " # your code here\n return True\n\n"} 146 | {"task_id": "HumanEval/145", "completion": " return sorted(nums, key=lambda x: (sum(int(i) for i in str(x)), x))\n\n"} 147 | {"task_id": "HumanEval/146", "completion": " count = 0\n for i in nums:\n if i > 10 and i % 2 != 0 and i % 10 != 0:\n count += 1\n return count\n"} 148 | {"task_id": "HumanEval/147", "completion": " return n * (n - 1) * (n - 2) // 6\n\n"} 149 | {"task_id": "HumanEval/148", "completion": " planets = (\"Mercury\", \"Venus\", \"Earth\", \"Mars\", \"Jupiter\", \"Saturn\", \"Uranus\", \"Neptune\")\n if planet1 not in planets or planet2 not in planets:\n return ()\n if planet1 == planet2:\n return ()\n if planet1 == \"Mercury\":\n if planet2 == \"Venus\":\n return (\"Venus\")\n elif planet2 == \"Earth\":\n return (\"Venus\", \"Earth\")\n elif planet2 == \"Mars\":\n return (\"Venus\", \"Earth\", \"Mars\")\n elif planet2 == \"Jupiter\":\n return (\"Venus\", \"Earth\", \"Mars\", \"Jupiter\")\n elif planet2 == \"Saturn\":\n return (\"Venus\", \"Earth\", \"Mars\", \"Jupiter\", \"Saturn\")\n elif planet2 == \"Uranus\":\n return (\"Venus\", \"Earth\", \"Mars\", \"Jupiter\", \"Saturn\", \"Uranus\")\n elif planet2 == \"Neptune\":\n return (\"Venus\", \"Earth\", \"Mars\", \"Jupiter\", \"Saturn\", \"Uranus\", \"Neptune\")\n elif planet1 == \"Venus\":\n if planet2 == \"Earth\":\n return (\"Venus\", \"Earth\")\n elif planet2 == \"Mars\":\n return (\"Venus\", \"Earth\", \"Mars\")\n elif planet2 == \"Jupiter\":\n return (\"Venus\", \"Earth\", \"Mars\", \"Jupiter\")\n elif planet2 == \"Saturn\":\n return (\"Venus\", \"Earth\", \"Mars\", \"Jupiter\", \"Saturn\")\n elif planet2 == \"Uranus\":\n return (\"Venus\", \"Earth\", \"Mars\", \"Jupiter\", \"Saturn\", \"Uranus\")\n elif planet2 == \"Neptune\":\n return (\"Venus\", \"Earth\", \"Mars\", \"Jupiter\", \"Saturn\", \"Uranus\", \"Neptune\")\n elif planet1 == \"Earth\":\n if planet2 == \"Mars\":\n return (\"Venus\", \"Earth\", \"Mars\")\n elif planet2 =="} 150 | {"task_id": "HumanEval/149", "completion": " lst.sort(key=len)\n return lst\n\n"} 151 | {"task_id": "HumanEval/150", "completion": " if is_prime(n):\n return x\n else:\n return y\n"} 152 | {"task_id": "HumanEval/151", "completion": " sum = 0\n for num in lst:\n if num % 2 != 0:\n sum += num ** 2\n return sum\n"} 153 | {"task_id": "HumanEval/152", "completion": " return [abs(guess[i]-game[i]) for i in range(len(guess))]"} 154 | {"task_id": "HumanEval/153", "completion": " # TODO: Implement this function\n #raise NotImplementedError\n #return 'my_class.AA'\n strength = []\n for extension in extensions:\n strength.append(Strength(extension))\n max_strength = max(strength)\n max_strength_index = strength.index(max_strength)\n return class_name + '.' + extensions[max_strength_index]\n"} 155 | {"task_id": "HumanEval/154", "completion": " if a == b:\n return True\n if len(a) < len(b):\n return False\n if len(b) == 1:\n return a.count(b) > 0\n if len(b) == 2:\n return a.count(b) > 0 or a.count(b[1] + b[0]) > 0\n if len(b) > 2:\n for i in range(len(b)):\n if a.count(b[i:] + b[:i]) > 0:\n return True\n return False\n\n"} 156 | {"task_id": "HumanEval/155", "completion": " even = 0\n odd = 0\n while num > 0:\n if num % 2 == 0:\n even += 1\n else:\n odd += 1\n num = num // 10\n return (even, odd)\n\n"} 157 | {"task_id": "HumanEval/156", "completion": " # TODO: Implement this function!\n pass\n\n"} 158 | {"task_id": "HumanEval/157", "completion": " if a**2 + b**2 == c**2:\n return True\n elif a**2 + c**2 == b**2:\n return True\n elif b**2 + c**2 == a**2:\n return True\n else:\n return False\n"} 159 | {"task_id": "HumanEval/158", "completion": " # your code here\n max_word = \"\"\n max_len = 0\n for word in words:\n if len(word) > max_len:\n max_len = len(word)\n max_word = word\n elif len(word) == max_len:\n if word < max_word:\n max_word = word\n return max_word\n\n"} 160 | {"task_id": "HumanEval/159", "completion": " return [number + need, remaining - need]\n"} 161 | {"task_id": "HumanEval/160", "completion": " result = 0\n for i in range(len(operator)):\n if operator[i] == '+':\n result += operand[i]\n elif operator[i] == '-':\n result -= operand[i]\n elif operator[i] == '*':\n result *= operand[i]\n elif operator[i] == '//':\n result //= operand[i]\n elif operator[i] == '**':\n result **= operand[i]\n return result\n\n"} 162 | {"task_id": "HumanEval/161", "completion": " res = ''\n for c in s:\n if c.isalpha():\n res += c.upper() if c.islower() else c.lower()\n else:\n res += c\n return res\n\n"} 163 | {"task_id": "HumanEval/162", "completion": " if text == '':\n return None\n else:\n return hashlib.md5(text).hexdigest()\n\n"} 164 | {"task_id": "HumanEval/163", "completion": " return [i for i in range(a, b + 1) if i % 2 == 0]\n\n"} 165 | -------------------------------------------------------------------------------- /data/results-code-davinci-002-1.jsonl: -------------------------------------------------------------------------------- 1 | {"task_id": "HumanEval/0", "completion": " for i in range(len(numbers)):\n for j in range(i + 1, len(numbers)):\n if abs(numbers[i] - numbers[j]) < threshold:\n return True\n return False\n\n"} 2 | {"task_id": "HumanEval/1", "completion": " # TODO: implement this function\n pass\n\n"} 3 | {"task_id": "HumanEval/2", "completion": " return number - int(number)\n\n"} 4 | {"task_id": "HumanEval/3", "completion": " balance = 0\n for operation in operations:\n balance += operation\n if balance < 0:\n return True\n return False\n\n"} 5 | {"task_id": "HumanEval/4", "completion": " mean = sum(numbers) / len(numbers)\n return sum([abs(x - mean) for x in numbers]) / len(numbers)\n\n"} 6 | {"task_id": "HumanEval/5", "completion": " if not numbers:\n return []\n if len(numbers) == 1:\n return numbers\n return [numbers[0], delimeter] + intersperse(numbers[1:], delimeter)\n\n"} 7 | {"task_id": "HumanEval/6", "completion": " max_nesting = 0\n current_nesting = 0\n max_nesting_list = []\n for char in paren_string:\n if char == '(':\n current_nesting += 1\n max_nesting = max(max_nesting, current_nesting)\n elif char == ')':\n current_nesting -= 1\n elif char == ' ':\n max_nesting_list.append(max_nesting)\n max_nesting = 0\n current_nesting = 0\n max_nesting_list.append(max_nesting)\n return max_nesting_list"} 8 | {"task_id": "HumanEval/7", "completion": " return [s for s in strings if substring in s]\n\n"} 9 | {"task_id": "HumanEval/8", "completion": " sum_ = 0\n product = 1\n for number in numbers:\n sum_ += number\n product *= number\n return sum_, product\n\n"} 10 | {"task_id": "HumanEval/9", "completion": " result = []\n max_value = numbers[0]\n for n in numbers:\n if n > max_value:\n max_value = n\n result.append(max_value)\n return result\n\n"} 11 | {"task_id": "HumanEval/10", "completion": " if not string:\n return string\n for i in range(len(string)):\n if is_palindrome(string[i:]):\n return string + string[:i][::-1]\n return string + string[::-1]\n\n"} 12 | {"task_id": "HumanEval/11", "completion": " return ''.join(str(int(x) ^ int(y)) for x, y in zip(a, b))\n\n"} 13 | {"task_id": "HumanEval/12", "completion": " if not strings:\n return None\n longest_string = strings[0]\n for string in strings:\n if len(string) > len(longest_string):\n longest_string = string\n return longest_string\n\n"} 14 | {"task_id": "HumanEval/13", "completion": " while a != b:\n if a > b:\n a = a - b\n else:\n b = b - a\n return a\n\n"} 15 | {"task_id": "HumanEval/14", "completion": " return [string[:i] for i in range(1, len(string) + 1)]\n\n"} 16 | {"task_id": "HumanEval/15", "completion": " return \" \".join(str(i) for i in range(n + 1))\n\n"} 17 | {"task_id": "HumanEval/16", "completion": " return len(set(string.lower()))\n\n"} 18 | {"task_id": "HumanEval/17", "completion": " return [4 if x == 'o' else 2 if x == 'o|' else 1 if x == '.|' else 0 for x in music_string.split()]\n\n"} 19 | {"task_id": "HumanEval/18", "completion": " count = 0\n for i in range(len(string)):\n if string[i:].startswith(substring):\n count += 1\n return count\n\n"} 20 | {"task_id": "HumanEval/19", "completion": " nums = numbers.split()\n nums.sort(key=lambda x: int(x))\n return ' '.join(nums)\n\n"} 21 | {"task_id": "HumanEval/20", "completion": " # TODO: Implement this function\n pass\n\n"} 22 | {"task_id": "HumanEval/21", "completion": " if len(numbers) < 2:\n raise ValueError(\"List must have at least two elements\")\n\n min_value = min(numbers)\n max_value = max(numbers)\n if min_value == max_value:\n return [0.5] * len(numbers)\n\n return [(x - min_value) / (max_value - min_value) for x in numbers]\n\n"} 23 | {"task_id": "HumanEval/22", "completion": " return [x for x in values if isinstance(x, int)]\n\n"} 24 | {"task_id": "HumanEval/23", "completion": " return len(string)\n\n"} 25 | {"task_id": "HumanEval/24", "completion": " for i in range(n - 1, 0, -1):\n if n % i == 0:\n return i\n return 1\n\n"} 26 | {"task_id": "HumanEval/25", "completion": " factors = []\n i = 2\n while n > 1:\n while n % i == 0:\n factors.append(i)\n n //= i\n i += 1\n return factors\n\n"} 27 | {"task_id": "HumanEval/26", "completion": " return list(dict.fromkeys(numbers))\n\n"} 28 | {"task_id": "HumanEval/27", "completion": " return string.swapcase()\n\n"} 29 | {"task_id": "HumanEval/28", "completion": " return ''.join(strings)\n\n"} 30 | {"task_id": "HumanEval/29", "completion": " return [s for s in strings if s.startswith(prefix)]\n\n"} 31 | {"task_id": "HumanEval/30", "completion": " return [x for x in l if x > 0]\n\n"} 32 | {"task_id": "HumanEval/31", "completion": " if n == 1:\n return False\n elif n == 2:\n return True\n else:\n for i in range(2, n):\n if n % i == 0:\n return False\n return True\n\n"} 33 | {"task_id": "HumanEval/32", "completion": " assert len(xs) % 2 == 0, \"xs must have even number of coefficients\"\n assert xs[0] == 0, \"xs must have largest non zero coefficient\"\n return -xs[1] / xs[2]\n\n"} 34 | {"task_id": "HumanEval/33", "completion": " l_copy = l[:]\n for i in range(0, len(l), 3):\n l_copy[i:i+3] = sorted(l[i:i+3])\n return l_copy\n"} 35 | {"task_id": "HumanEval/34", "completion": " return sorted(set(l))\n\n"} 36 | {"task_id": "HumanEval/35", "completion": " max_value = l[0]\n for i in range(1, len(l)):\n if l[i] > max_value:\n max_value = l[i]\n return max_value\n\n"} 37 | {"task_id": "HumanEval/36", "completion": " count = 0\n for i in range(n):\n if i % 11 == 0 or i % 13 == 0:\n count += str(i).count('7')\n return count\n"} 38 | {"task_id": "HumanEval/37", "completion": " l_sorted = []\n for i in range(len(l)):\n if i % 2 == 0:\n l_sorted.append(l[i])\n else:\n l_sorted.append(sorted(l[i])[0])\n return l_sorted\n"} 39 | {"task_id": "HumanEval/38", "completion": " # split string to groups. Each of length 3.\n groups = [s[(3 * i):min((3 * i + 3), len(s))] for i in range((len(s) + 2) // 3)]\n # cycle elements in each group. Unless group has fewer elements than 3.\n groups = [(group[2:] + group[:2]) if len(group) == 3 else group for group in groups]\n return \"\".join(groups)\n\n"} 40 | {"task_id": "HumanEval/39", "completion": " fib_list = [2, 3]\n i = 2\n while i <= n:\n fib_list.append(fib_list[i - 1] + fib_list[i - 2])\n i += 1\n return fib_list[n - 1]\n\n"} 41 | {"task_id": "HumanEval/40", "completion": " for i in range(len(l)):\n for j in range(i + 1, len(l)):\n for k in range(j + 1, len(l)):\n if l[i] + l[j] + l[k] == 0:\n return True\n return False\n\n"} 42 | {"task_id": "HumanEval/41", "completion": " # TODO: implement this function\n pass\n"} 43 | {"task_id": "HumanEval/42", "completion": " return [i + 1 for i in l]\n\n"} 44 | {"task_id": "HumanEval/43", "completion": " for i in range(len(l)):\n for j in range(i+1, len(l)):\n if l[i] + l[j] == 0:\n return True\n return False\n"} 45 | {"task_id": "HumanEval/44", "completion": " if x == 0:\n return '0'\n result = ''\n while x > 0:\n result = str(x % base) + result\n x = x // base\n return result\n\n"} 46 | {"task_id": "HumanEval/45", "completion": " return (a * h) / 2\n\n"} 47 | {"task_id": "HumanEval/46", "completion": " if n == 0:\n return 0\n elif n == 1:\n return 0\n elif n == 2:\n return 2\n elif n == 3:\n return 0\n else:\n return fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4)\n"} 48 | {"task_id": "HumanEval/47", "completion": " l.sort()\n if len(l) % 2 == 0:\n return (l[len(l) // 2] + l[len(l) // 2 - 1]) / 2\n else:\n return l[len(l) // 2]\n\n"} 49 | {"task_id": "HumanEval/48", "completion": " return text == text[::-1]\n\n"} 50 | {"task_id": "HumanEval/49", "completion": " if n == 0:\n return 1\n elif n % 2 == 0:\n return modp(n // 2, p) ** 2 % p\n else:\n return 2 * modp(n - 1, p) % p\n\n"} 51 | {"task_id": "HumanEval/50", "completion": " return \"\".join([chr(((ord(ch) - 5 - ord(\"a\")) % 26) + ord(\"a\")) for ch in s])\n\n"} 52 | {"task_id": "HumanEval/51", "completion": " return ''.join(c for c in text if c not in 'aeiouAEIOU')\n\n"} 53 | {"task_id": "HumanEval/52", "completion": " for i in l:\n if i >= t:\n return False\n return True\n"} 54 | {"task_id": "HumanEval/53", "completion": " return x + y\n"} 55 | {"task_id": "HumanEval/54", "completion": " return sorted(s0) == sorted(s1)\n\n"} 56 | {"task_id": "HumanEval/55", "completion": " if n == 1:\n return 1\n elif n == 2:\n return 1\n else:\n return fib(n-1) + fib(n-2)\n"} 57 | {"task_id": "HumanEval/56", "completion": " count = 0\n for char in brackets:\n if char == \"<\":\n count += 1\n elif char == \">\":\n count -= 1\n if count < 0:\n return False\n return count == 0\n"} 58 | {"task_id": "HumanEval/57", "completion": " return all(l[i] <= l[i+1] for i in range(len(l)-1)) or all(l[i] >= l[i+1] for i in range(len(l)-1))\n"} 59 | {"task_id": "HumanEval/58", "completion": " return sorted(set(l1) & set(l2))\n\n"} 60 | {"task_id": "HumanEval/59", "completion": " assert n > 1 and not is_prime(n)\n for i in range(2, n):\n if n % i == 0 and is_prime(i):\n return i\n"} 61 | {"task_id": "HumanEval/60", "completion": " return n * (n + 1) // 2\n\n"} 62 | {"task_id": "HumanEval/61", "completion": " count = 0\n for i in brackets:\n if i == \"(\":\n count += 1\n elif i == \")\":\n count -= 1\n if count < 0:\n return False\n return count == 0\n"} 63 | {"task_id": "HumanEval/62", "completion": " return [xs[i] * (len(xs) - i - 1) for i in range(len(xs) - 1)]\n\n"} 64 | {"task_id": "HumanEval/63", "completion": " if n == 0:\n return 0\n elif n == 1:\n return 0\n elif n == 2:\n return 1\n else:\n return fibfib(n-1) + fibfib(n-2) + fibfib(n-3)\n"} 65 | {"task_id": "HumanEval/64", "completion": " return sum(1 for c in s.lower() if c in 'aeiouy')\n"} 66 | {"task_id": "HumanEval/65", "completion": " x = str(x)\n if shift > len(x):\n return x[::-1]\n return x[-shift:] + x[:-shift]\n\n"} 67 | {"task_id": "HumanEval/66", "completion": " return sum(ord(c) for c in s if c.isupper())\n\n"} 68 | {"task_id": "HumanEval/67", "completion": " # your code here\n pass\n"} 69 | {"task_id": "HumanEval/68", "completion": " evens = [x for x in arr if x % 2 == 0]\n if evens:\n return [min(evens), arr.index(min(evens))]\n return []\n"} 70 | {"task_id": "HumanEval/69", "completion": " # your code here\n pass\n\n"} 71 | {"task_id": "HumanEval/70", "completion": " if not lst:\n return []\n lst = sorted(lst)\n res = []\n while lst:\n res.append(lst.pop(0))\n if lst:\n res.append(lst.pop())\n return res\n\n"} 72 | {"task_id": "HumanEval/71", "completion": " if a + b > c and a + c > b and b + c > a:\n p = (a + b + c) / 2\n return round((p * (p - a) * (p - b) * (p - c)) ** 0.5, 2)\n else:\n return -1\n"} 73 | {"task_id": "HumanEval/72", "completion": " if sum(q) <= w and q == q[::-1]:\n return True\n else:\n return False\n"} 74 | {"task_id": "HumanEval/73", "completion": " if len(arr) < 2:\n return 0\n\n # find the longest palindrome\n longest_palindrome = longest_palindrome_subsequence(arr)\n\n # return the difference between the length of the array and the longest palindrome\n return len(arr) - len(longest_palindrome)\n\n"} 75 | {"task_id": "HumanEval/74", "completion": " if len(lst1) == 0 and len(lst2) == 0:\n return []\n elif len(lst1) == 0:\n return lst2\n elif len(lst2) == 0:\n return lst1\n else:\n lst1_total = 0\n lst2_total = 0\n for i in lst1:\n lst1_total += len(i)\n for i in lst2:\n lst2_total += len(i)\n if lst1_total < lst2_total:\n return lst1\n elif lst1_total > lst2_total:\n return lst2\n else:\n return lst1\n"} 76 | {"task_id": "HumanEval/75", "completion": " pass\n"} 77 | {"task_id": "HumanEval/76", "completion": " return x == n ** int(math.log(x, n))\n\n"} 78 | {"task_id": "HumanEval/77", "completion": " return a == 0 or a == 1 or (a < 0 and a % 2 == 0) or (a > 0 and a % 2 == 1)\n"} 79 | {"task_id": "HumanEval/78", "completion": " primes = [2, 3, 5, 7, 11, 13]\n return len([x for x in num if int(x, 16) in primes])\n"} 80 | {"task_id": "HumanEval/79", "completion": " return 'db' + bin(decimal)[2:] + 'db'\n\n"} 81 | {"task_id": "HumanEval/80", "completion": " if len(s) < 3:\n return False\n for i in range(len(s) - 2):\n if s[i] == s[i + 1] or s[i] == s[i + 2] or s[i + 1] == s[i + 2]:\n return False\n return True\n\n"} 82 | {"task_id": "HumanEval/81", "completion": " # your code here\n grades_list = []\n for grade in grades:\n if grade == 4.0:\n grades_list.append('A+')\n elif grade > 3.7:\n grades_list.append('A')\n elif grade > 3.3:\n grades_list.append('A-')\n elif grade > 3.0:\n grades_list.append('B+')\n elif grade > 2.7:\n grades_list.append('B')\n elif grade > 2.3:\n grades_list.append('B-')\n elif grade > 2.0:\n grades_list.append('C+')\n elif grade > 1.7:\n grades_list.append('C')\n elif grade > 1.3:\n grades_list.append('C-')\n elif grade > 1.0:\n grades_list.append('D+')\n elif grade > 0.7:\n grades_list.append('D')\n elif grade > 0.0:\n grades_list.append('D-')\n elif grade == 0.0:\n grades_list.append('E')\n return grades_list\n"} 83 | {"task_id": "HumanEval/82", "completion": " return is_prime(len(string))\n\n"} 84 | {"task_id": "HumanEval/83", "completion": " if n == 1:\n return 2\n else:\n return starts_one_ends(n - 1) + 9 * (10 ** (n - 2))\n\n"} 85 | {"task_id": "HumanEval/84", "completion": " # convert N to binary\n binary = bin(N)\n # remove the '0b' prefix\n binary = binary[2:]\n # sum the digits\n sum_digits = sum([int(i) for i in binary])\n # return the sum\n return sum_digits\n"} 86 | {"task_id": "HumanEval/85", "completion": " return sum([lst[i] for i in range(1, len(lst), 2) if lst[i] % 2 == 0])\n\n"} 87 | {"task_id": "HumanEval/86", "completion": " return ' '.join([''.join(sorted(word)) for word in s.split()])\n\n"} 88 | {"task_id": "HumanEval/87", "completion": " res = []\n for i in range(len(lst)):\n for j in range(len(lst[i])):\n if lst[i][j] == x:\n res.append((i, j))\n res.sort(key=lambda x: (x[0], -x[1]))\n return res\n"} 89 | {"task_id": "HumanEval/88", "completion": " if len(array) == 0:\n return array\n if len(array) == 1:\n return array\n if len(array) == 2:\n if array[0] > array[1]:\n return [array[1], array[0]]\n else:\n return array\n if (array[0] + array[-1]) % 2 == 0:\n return sorted(array, reverse=True)\n else:\n return sorted(array)\n\n"} 90 | {"task_id": "HumanEval/89", "completion": " # TODO: Implement this function\n # Hint: You can use the ord() and chr() functions to convert between\n # characters and integers.\n # Hint: You can use the % operator to wrap around the alphabet.\n # Hint: You can use the len() function to find the length of a string.\n # Hint: You can use the range() function to iterate over a string.\n # Hint: You can use the join() function to join a list of strings.\n # Hint: You can use the string.ascii_lowercase variable to get the\n # lowercase alphabet.\n # Hint: You can use the string.ascii_uppercase variable to get the\n # uppercase alphabet.\n # Hint: You can use the string.ascii_letters variable to get the\n # lowercase and uppercase alphabet.\n # Hint: You can use the string.digits variable to get the digits.\n # Hint: You can use the string.punctuation variable to get the\n # punctuation.\n # Hint: You can use the string.printable variable to get all the\n # printable characters.\n # Hint: You can use the string.whitespace variable to get the\n # whitespace characters.\n # Hint: You can use the string.hexdigits variable to get the\n # hexadecimal digits.\n # Hint: You can use the string.octdigits variable to get the\n # octal digits.\n # Hint: You can use the string.ascii_lowercase[::-1] to get the\n # lowercase alphabet in reverse.\n # Hint: You can use the string.ascii_uppercase[::-1] to get the\n # uppercase alphabet in reverse.\n # Hint: You can use the string.ascii_letters[::-1] to get the\n # lowercase and uppercase alphabet in reverse.\n # Hint: You can use the string.digits[::-1] to get the digits in\n # reverse.\n "} 91 | {"task_id": "HumanEval/90", "completion": " if len(lst) < 2:\n return None\n else:\n lst.sort()\n for i in range(len(lst)):\n if lst[i] != lst[i+1]:\n return lst[i+1]\n else:\n return None\n"} 92 | {"task_id": "HumanEval/91", "completion": " boredoms = 0\n sentences = S.split('.')\n for sentence in sentences:\n if sentence.split()[0] == 'I':\n boredoms += 1\n return boredoms\n"} 93 | {"task_id": "HumanEval/92", "completion": " # your code here\n if (x + y == z) or (x + z == y) or (y + z == x):\n return True\n else:\n return False\n"} 94 | {"task_id": "HumanEval/93", "completion": " # TODO: Write your code here\n pass\n"} 95 | {"task_id": "HumanEval/94", "completion": " lst = sorted(lst)\n lst = lst[::-1]\n for i in lst:\n if is_prime(i):\n return sum(int(x) for x in str(i))\n return 0\n\n"} 96 | {"task_id": "HumanEval/95", "completion": " if len(dict) == 0:\n return False\n else:\n for key in dict:\n if not isinstance(key, str):\n return False\n if key.islower() or key.isupper():\n return True\n else:\n return False\n"} 97 | {"task_id": "HumanEval/96", "completion": " if n < 2:\n return []\n primes = [2]\n x = 3\n while x <= n:\n for y in range(3,x,2): # test all odd factors up to x-1\n if x%y == 0:\n x += 2\n break\n else:\n primes.append(x)\n x += 2\n return primes\n"} 98 | {"task_id": "HumanEval/97", "completion": " return (a % 10) * (b % 10)\n"} 99 | {"task_id": "HumanEval/98", "completion": " # YOUR CODE HERE\n count = 0\n for i in range(len(s)):\n if i % 2 == 0 and s[i].isupper():\n count += 1\n return count"} 100 | {"task_id": "HumanEval/99", "completion": " return int(round(float(value)))\n"} 101 | {"task_id": "HumanEval/100", "completion": " # YOUR CODE HERE\n raise NotImplementedError()\n\n"} 102 | {"task_id": "HumanEval/101", "completion": " return s.split()\n"} 103 | {"task_id": "HumanEval/102", "completion": " # YOUR CODE HERE\n if x%2 == 0:\n return x\n elif x%2 != 0:\n return x+1\n else:\n return -1\n"} 104 | {"task_id": "HumanEval/103", "completion": " if n > m:\n return -1\n else:\n return bin(int(round((n + m) / 2)))\n"} 105 | {"task_id": "HumanEval/104", "completion": " return sorted([i for i in x if not any(map(lambda x: int(x) % 2 == 0, str(i)))])\n\n"} 106 | {"task_id": "HumanEval/105", "completion": " # YOUR CODE HERE\n if len(arr) == 0:\n return []\n else:\n arr.sort()\n arr.reverse()\n for i in range(len(arr)):\n if arr[i] > 9 or arr[i] < 1:\n arr[i] = 0\n arr = [x for x in arr if x != 0]\n for i in range(len(arr)):\n if arr[i] == 1:\n arr[i] = \"One\"\n elif arr[i] == 2:\n arr[i] = \"Two\"\n elif arr[i] == 3:\n arr[i] = \"Three\"\n elif arr[i] == 4:\n arr[i] = \"Four\"\n elif arr[i] == 5:\n arr[i] = \"Five\"\n elif arr[i] == 6:\n arr[i] = \"Six\"\n elif arr[i] == 7:\n arr[i] = \"Seven\"\n elif arr[i] == 8:\n arr[i] = \"Eight\"\n elif arr[i] == 9:\n arr[i] = \"Nine\"\n return arr\n raise NotImplementedError()\n\n"} 107 | {"task_id": "HumanEval/106", "completion": " return [i if i % 2 else factorial(i) for i in range(1, n + 1)]\n\n"} 108 | {"task_id": "HumanEval/107", "completion": " even_count = 0\n odd_count = 0\n\n for i in range(1, n + 1):\n if is_palindrome(i):\n if i % 2 == 0:\n even_count += 1\n else:\n odd_count += 1\n\n return even_count, odd_count\n\n"} 109 | {"task_id": "HumanEval/108", "completion": " return sum(1 for x in arr if sum(map(abs, digits(x))) > 0)\n\n"} 110 | {"task_id": "HumanEval/109", "completion": " if len(arr) == 0:\n return True\n if len(arr) == 1:\n return True\n if len(arr) == 2:\n return True\n if len(arr) == 3:\n if arr[0] > arr[1] and arr[1] > arr[2]:\n return True\n else:\n return False\n if len(arr) == 4:\n if arr[0] > arr[1] and arr[1] > arr[2] and arr[2] > arr[3]:\n return True\n else:\n return False\n if len(arr) == 5:\n if arr[0] > arr[1] and arr[1] > arr[2] and arr[2] > arr[3] and arr[3] > arr[4]:\n return True\n else:\n return False\n if len(arr) == 6:\n if arr[0] > arr[1] and arr[1] > arr[2] and arr[2] > arr[3] and arr[3] > arr[4] and arr[4] > arr[5]:\n return True\n else:\n return False\n if len(arr) == 7:\n if arr[0] > arr[1] and arr[1] > arr[2] and arr[2] > arr[3] and arr[3] > arr[4] and arr[4] > arr[5] and arr[5] > arr[6]:\n return True\n else:\n return False\n if len(arr) == 8:\n if arr[0] > arr[1] and arr[1] > arr[2] and arr[2] > arr[3] and arr[3] > arr[4] and arr[4] > arr[5] and arr[5] > arr[6] and arr[6] > arr[7]:\n return True\n else:\n return False\n if len(arr) == 9:\n if arr[0] > arr[1] and arr[1] > arr[2] and arr[2] > arr[3] and arr[3] > arr[4] and arr[4"} 111 | {"task_id": "HumanEval/110", "completion": " # Your code here\n # return \"YES\" or \"NO\"\n lst1_even = [i for i in lst1 if i % 2 == 0]\n lst1_odd = [i for i in lst1 if i % 2 != 0]\n lst2_even = [i for i in lst2 if i % 2 == 0]\n lst2_odd = [i for i in lst2 if i % 2 != 0]\n if len(lst1_even) == len(lst1):\n return \"YES\"\n elif len(lst1_even) + len(lst2_odd) >= len(lst1):\n return \"YES\"\n else:\n return \"NO\"\n\n"} 112 | {"task_id": "HumanEval/111", "completion": " # Create a dictionary of the letters and their counts\n hist = {}\n for letter in test:\n if letter in hist:\n hist[letter] += 1\n else:\n hist[letter] = 1\n \n # Find the highest count\n highest_count = 0\n for key in hist:\n if hist[key] > highest_count:\n highest_count = hist[key]\n \n # Create a dictionary of the letters with the highest count\n highest_count_dict = {}\n for key in hist:\n if hist[key] == highest_count:\n highest_count_dict[key] = highest_count\n \n return highest_count_dict\n"} 113 | {"task_id": "HumanEval/112", "completion": " for i in c:\n s = s.replace(i,'')\n return s, s == s[::-1]\n"} 114 | {"task_id": "HumanEval/113", "completion": " return [\"the number of odd elements {}n the str{}ng {} of the {}nput.\".format(\n len([x for x in i if int(x) % 2 == 1]),\n i.index(i),\n i.index(i),\n i.index(i)) for i in lst]\n\n"} 115 | {"task_id": "HumanEval/114", "completion": " min_sum = float('inf')\n curr_sum = 0\n for num in nums:\n curr_sum += num\n min_sum = min(min_sum, curr_sum)\n curr_sum = max(curr_sum, 0)\n return min_sum\n\n"} 116 | {"task_id": "HumanEval/115", "completion": " # get the number of wells\n n_wells = len(grid)\n # get the number of units of water in each well\n n_water = [sum(row) for row in grid]\n # get the number of buckets needed to fill each well\n n_buckets = [math.ceil(n / capacity) for n in n_water]\n # get the total number of buckets needed\n n_total = sum(n_buckets)\n # get the number of buckets needed to fill the first well\n n_first = n_buckets[0]\n # get the number of buckets needed to fill the last well\n n_last = n_buckets[-1]\n # get the number of buckets needed to fill the first and last wells\n n_first_last = n_first + n_last\n # get the number of buckets needed to fill the first and last wells\n # if the first and last wells are filled at the same time\n n_first_last_same = math.ceil((n_first + n_last) / capacity)\n # get the number of buckets needed to fill the first and last wells\n # if the first and last wells are filled at different times\n n_first_last_diff = n_first + n_last\n # get the minimum number of buckets needed to fill the first and last wells\n n_first_last_min = min(n_first_last_same, n_first_last_diff)\n # get the number of buckets needed to fill the first and last wells\n # if the first and last wells are filled at different times\n n_first_last_diff = n_first + n_last\n # get the minimum number of buckets needed to fill the first and last wells\n n_first_last_min = min(n_first_last_same, n_first_last_diff)\n # get the number of buckets needed to fill the first and last wells\n # if the first and last wells are filled at different times\n n_first_last_diff = n_first + n_last\n # get the minimum number of buckets needed to fill the first and last wells\n n_first_last_min = min(n"} 117 | {"task_id": "HumanEval/116", "completion": " return sorted(arr, key=lambda x: (bin(x).count('1'), x))\n\n"} 118 | {"task_id": "HumanEval/117", "completion": " # Your code here\n words = s.split()\n result = []\n for word in words:\n count = 0\n for letter in word:\n if letter not in \"aeiouAEIOU\":\n count += 1\n if count == n:\n result.append(word)\n return result\n"} 119 | {"task_id": "HumanEval/118", "completion": " vowels = \"aeiou\"\n for i in range(len(word)-1, 0, -1):\n if word[i] in vowels and word[i-1] not in vowels:\n return word[i]\n return \"\"\n\n"} 120 | {"task_id": "HumanEval/119", "completion": " if len(lst) != 2:\n raise ValueError('Invalid input')\n if len(lst[0]) == 0 and len(lst[1]) == 0:\n return 'Yes'\n if len(lst[0]) == 0:\n if lst[1][0] == '(':\n return 'No'\n else:\n return match_parens([lst[0], lst[1][1:]])\n if len(lst[1]) == 0:\n if lst[0][-1] == ')':\n return 'No'\n else:\n return match_parens([lst[0][:-1], lst[1]])\n if lst[0][-1] == ')' and lst[1][0] == '(':\n return 'Yes'\n if lst[0][-1] == ')':\n return match_parens([lst[0][:-1], lst[1]])\n if lst[1][0] == '(':\n return match_parens([lst[0], lst[1][1:]])\n return 'No'\n\n"} 121 | {"task_id": "HumanEval/120", "completion": " return sorted(arr, reverse=True)[:k]\n\n"} 122 | {"task_id": "HumanEval/121", "completion": " return sum(lst[i] for i in range(len(lst)) if i % 2 == 0 and lst[i] % 2 != 0)\n"} 123 | {"task_id": "HumanEval/122", "completion": " return sum(i for i in arr[:k] if i < 100)\n\n"} 124 | {"task_id": "HumanEval/123", "completion": " if n == 1:\n return [1]\n else:\n collatz = [n]\n while n != 1:\n if n % 2 == 0:\n n = n // 2\n else:\n n = 3 * n + 1\n collatz.append(n)\n return sorted([x for x in collatz if x % 2 != 0])\n\n"} 125 | {"task_id": "HumanEval/124", "completion": " # your code here\n if date == '':\n return False\n else:\n date = date.split('-')\n if len(date) != 3:\n return False\n else:\n month = int(date[0])\n day = int(date[1])\n year = int(date[2])\n if month < 1 or month > 12:\n return False\n elif month == 2:\n if day < 1 or day > 29:\n return False\n elif month == 1 or month == 3 or month == 5 or month == 7 or month == 8 or month == 10 or month == 12:\n if day < 1 or day > 31:\n return False\n elif month == 4 or month == 6 or month == 9 or month == 11:\n if day < 1 or day > 30:\n return False\n else:\n return True\n\n"} 126 | {"task_id": "HumanEval/125", "completion": " if ' ' in txt:\n return txt.split(' ')\n elif ',' in txt:\n return txt.split(',')\n else:\n return sum(1 for i in txt if ord(i) % 2 == 0)\n"} 127 | {"task_id": "HumanEval/126", "completion": " return lst == sorted(lst)\n"} 128 | {"task_id": "HumanEval/127", "completion": " start = max(interval1[0], interval2[0])\n end = min(interval1[1], interval2[1])\n if start > end:\n return \"NO\"\n return \"YES\" if is_prime(end - start + 1) else \"NO\"\n\n"} 129 | {"task_id": "HumanEval/128", "completion": " if not arr:\n return None\n signs = [1 if x >= 0 else -1 for x in arr]\n return sum(map(abs, arr)) * reduce(lambda x, y: x * y, signs)\n\n"} 130 | {"task_id": "HumanEval/129", "completion": " n = len(grid)\n m = len(grid[0])\n visited = [[False for _ in range(m)] for _ in range(n)]\n min_path = []\n min_sum = float('inf')\n for i in range(n):\n for j in range(m):\n path = []\n dfs(grid, i, j, k, path, visited)\n if sum(path) < min_sum:\n min_sum = sum(path)\n min_path = path\n return min_path\n"} 131 | {"task_id": "HumanEval/130", "completion": " if n == 0:\n return [1]\n if n == 1:\n return [1, 3]\n if n == 2:\n return [1, 3, 2]\n if n == 3:\n return [1, 3, 2, 8]\n if n == 4:\n return [1, 3, 2, 8, 5]\n if n == 5:\n return [1, 3, 2, 8, 5, 13]\n if n == 6:\n return [1, 3, 2, 8, 5, 13, 21]\n if n == 7:\n return [1, 3, 2, 8, 5, 13, 21, 34]\n if n == 8:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55]\n if n == 9:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55, 89]\n if n == 10:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55, 89, 144]\n if n == 11:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55, 89, 144, 233]\n if n == 12:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55, 89, 144, 233, 377]\n if n == 13:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55, 89, 144, 233, 377, 610]\n if n == 14:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987]\n if n == 15:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597]\n if n == 16:\n return [1, 3, 2, 8, 5, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584]\n if n == 17:\n return [1,"} 132 | {"task_id": "HumanEval/131", "completion": " # your code here\n result = 1\n for i in str(n):\n if int(i) % 2 != 0:\n result *= int(i)\n return result\n"} 133 | {"task_id": "HumanEval/132", "completion": " stack = []\n for char in string:\n if char == '[':\n stack.append(char)\n elif char == ']':\n if len(stack) == 0:\n return False\n else:\n stack.pop()\n return len(stack) != 0\n"} 134 | {"task_id": "HumanEval/133", "completion": " return sum(map(lambda x: x**2, map(lambda x: math.ceil(x), lst)))\n"} 135 | {"task_id": "HumanEval/134", "completion": " if txt == '':\n return False\n else:\n if txt[-1].isalpha() and txt[-2] == ' ':\n return True\n else:\n return False\n"} 136 | {"task_id": "HumanEval/135", "completion": " for i in range(len(arr) - 1, 0, -1):\n if arr[i] >= arr[i - 1]:\n return i - 1\n return -1\n\n"} 137 | {"task_id": "HumanEval/136", "completion": " # your code here\n a = None\n b = None\n for i in lst:\n if i < 0:\n if a == None:\n a = i\n elif i > a:\n a = i\n elif i > 0:\n if b == None:\n b = i\n elif i < b:\n b = i\n return (a, b)\n"} 138 | {"task_id": "HumanEval/137", "completion": " if a == b:\n return None\n elif a > b:\n return a\n else:\n return b\n\n"} 139 | {"task_id": "HumanEval/138", "completion": " # your code here\n return n % 2 == 0 and n > 7 and n < 28\n\n"} 140 | {"task_id": "HumanEval/139", "completion": " if n == 0:\n return 1\n else:\n return n * special_factorial(n-1)\n\n"} 141 | {"task_id": "HumanEval/140", "completion": " return re.sub(r'\\s+', '-', re.sub(r'\\s', '_', text))\n\n"} 142 | {"task_id": "HumanEval/141", "completion": " if len(re.findall(r'\\d', file_name)) > 3:\n return 'No'\n if len(re.findall(r'\\.', file_name)) != 1:\n return 'No'\n if not re.match(r'[a-zA-Z]', file_name.split('.')[0]):\n return 'No'\n if file_name.split('.')[1] not in ['txt', 'exe', 'dll']:\n return 'No'\n return 'Yes'\n"} 143 | {"task_id": "HumanEval/142", "completion": " sum = 0\n for i in range(len(lst)):\n if i % 3 == 0:\n sum += lst[i]**2\n elif i % 4 == 0:\n sum += lst[i]**3\n else:\n sum += lst[i]\n return sum\n\n"} 144 | {"task_id": "HumanEval/143", "completion": " return \" \".join(word for word in sentence.split() if is_prime(len(word)))\n\n"} 145 | {"task_id": "HumanEval/144", "completion": " x = Fraction(x)\n n = Fraction(n)\n return x * n == int(x * n)\n\n"} 146 | {"task_id": "HumanEval/145", "completion": " return sorted(nums, key=lambda x: (sum(map(int, str(abs(x)))), nums.index(x)))\n\n"} 147 | {"task_id": "HumanEval/146", "completion": " count = 0\n for i in nums:\n if (i > 10) and (i % 2 != 0) and (i // 10 % 2 != 0):\n count += 1\n return count\n"} 148 | {"task_id": "HumanEval/147", "completion": " a = [i * i - i + 1 for i in range(1, n + 1)]\n count = 0\n for i in range(n - 2):\n for j in range(i + 1, n - 1):\n for k in range(j + 1, n):\n if (a[i] + a[j] + a[k]) % 3 == 0:\n count += 1\n return count\n"} 149 | {"task_id": "HumanEval/148", "completion": " planets = (\"Mercury\", \"Venus\", \"Earth\", \"Mars\", \"Jupiter\", \"Saturn\", \"Uranus\", \"Neptune\")\n if planet1 not in planets or planet2 not in planets:\n return ()\n else:\n return tuple(planets[planets.index(planet1)+1:planets.index(planet2)])\n"} 150 | {"task_id": "HumanEval/149", "completion": " return sorted(lst, key=lambda x: (len(x), x))\n\n"} 151 | {"task_id": "HumanEval/150", "completion": " if is_prime(n):\n return x\n else:\n return y\n"} 152 | {"task_id": "HumanEval/151", "completion": " # YOUR CODE HERE\n pass\n"} 153 | {"task_id": "HumanEval/152", "completion": " return [abs(game[i]-guess[i]) for i in range(len(game))]\n"} 154 | {"task_id": "HumanEval/153", "completion": " # your code here\n return class_name + '.' + max(extensions, key=lambda x: len([i for i in x if i.isupper()]) - len([i for i in x if i.islower()]))\n\n"} 155 | {"task_id": "HumanEval/154", "completion": " if a.find(b) != -1:\n return True\n else:\n for i in range(len(b)):\n b = b[1:] + b[0]\n if a.find(b) != -1:\n return True\n return False\n"} 156 | {"task_id": "HumanEval/155", "completion": " even = 0\n odd = 0\n for i in str(num):\n if int(i) % 2 == 0:\n even += 1\n else:\n odd += 1\n return (even, odd)\n\n"} 157 | {"task_id": "HumanEval/156", "completion": " roman_numerals = {\n 1: 'i',\n 4: 'iv',\n 5: 'v',\n 9: 'ix',\n 10: 'x',\n 40: 'xl',\n 50: 'l',\n 90: 'xc',\n 100: 'c',\n 400: 'cd',\n 500: 'd',\n 900: 'cm',\n 1000: 'm'\n }\n result = ''\n for key in sorted(roman_numerals.keys(), reverse=True):\n while number >= key:\n result += roman_numerals[key]\n number -= key\n return result\n\n"} 158 | {"task_id": "HumanEval/157", "completion": " if a**2 + b**2 == c**2:\n return True\n else:\n return False\n"} 159 | {"task_id": "HumanEval/158", "completion": " max_word = \"\"\n max_len = 0\n for word in words:\n if len(set(word)) > max_len:\n max_len = len(set(word))\n max_word = word\n return max_word\n\n"} 160 | {"task_id": "HumanEval/159", "completion": " # your code here\n return [number + min(need, remaining), max(remaining - need, 0)]\n"} 161 | {"task_id": "HumanEval/160", "completion": " result = operand[0]\n for i in range(len(operator)):\n if operator[i] == '+':\n result += operand[i+1]\n elif operator[i] == '-':\n result -= operand[i+1]\n elif operator[i] == '*':\n result *= operand[i+1]\n elif operator[i] == '//':\n result //= operand[i+1]\n elif operator[i] == '**':\n result **= operand[i+1]\n return result\n"} 162 | {"task_id": "HumanEval/161", "completion": " if not s:\n return s\n if s.isalpha():\n return s[::-1].swapcase()\n return ''.join(i.swapcase() if i.isalpha() else i for i in s[::-1])\n\n"} 163 | {"task_id": "HumanEval/162", "completion": " if text == '':\n return None\n else:\n return hashlib.md5(text.encode('utf-8')).hexdigest()\n\n"} 164 | {"task_id": "HumanEval/163", "completion": " return [x for x in range(min(a, b), max(a, b) + 1) if x % 2 == 0]\n\n"} 165 | -------------------------------------------------------------------------------- /gpt4/results-531.jsonl: -------------------------------------------------------------------------------- 1 | {"task_id": "HumanEval/7", "completion": "\n return [string for string in strings if substring in string]"} 2 | {"task_id": "HumanEval/8", "completion": "\n total_sum = 0\n total_product = 1\n\n for number in numbers:\n total_sum += number\n total_product *= number\n\n return total_sum, total_product"} 3 | {"task_id": "HumanEval/80", "completion": "\n if len(s) < 3:\n return False\n\n for i in range(len(s) - 2):\n if s[i] == s[i + 1] or s[i] == s[i + 2] or s[i + 1] == s[i + 2]:\n return False\n\n return True"} 4 | {"task_id": "HumanEval/152", "completion": "\n result = []\n for i in range(len(game)):\n difference = abs(game[i] - guess[i])\n result.append(difference)\n return result"} 5 | {"task_id": "HumanEval/83", "completion": "\n if n == 1:\n return 2 # 1 and 11\n else:\n return 2 * (10 ** (n - 1)) - 10 ** (n - 2)"} 6 | {"task_id": "HumanEval/9", "completion": "\n result = []\n current_max = float('-inf')\n for num in numbers:\n current_max = max(current_max, num)\n result.append(current_max)\n return result"} 7 | {"task_id": "HumanEval/11", "completion": "\n result = ''.join(str(int(x) ^ int(y)) for x, y in zip(a, b))\n return result"} 8 | {"task_id": "HumanEval/85", "completion": "\n result = 0\n for i in range(1, len(lst), 2):\n if lst[i] % 2 == 0:\n result += lst[i]\n return result"} 9 | {"task_id": "HumanEval/153", "completion": "\n def strength(extension):\n CAP = sum(1 for c in extension if c.isupper())\n SM = sum(1 for c in extension if c.islower())\n return CAP - SM\n\n strongest_extension = max(extensions, key=strength)\n return f\"{class_name}.{strongest_extension}\""} 10 | {"task_id": "HumanEval/155", "completion": "\n even_count = 0\n odd_count = 0\n\n # Convert the number to its absolute value to handle negative numbers\n num = abs(num)\n\n # Iterate through each digit in the number\n for digit in str(num):\n # Check if the digit is even or odd and increment the respective count\n if int(digit) % 2 == 0:\n even_count += 1\n else:\n odd_count += 1\n\n # Return the tuple with the even and odd counts\n return (even_count, odd_count)"} 11 | {"task_id": "HumanEval/13", "completion": "\n while b:\n a, b = b, a % b\n return a"} 12 | {"task_id": "HumanEval/81", "completion": "\n letter_grades = []\n for gpa in grades:\n if gpa == 4.0:\n letter_grades.append(\"A+\")\n elif gpa > 3.7:\n letter_grades.append(\"A\")\n elif gpa > 3.3:\n letter_grades.append(\"A-\")\n elif gpa > 3.0:\n letter_grades.append(\"B+\")\n elif gpa > 2.7:\n letter_grades.append(\"B\")\n elif gpa > 2.3:\n letter_grades.append(\"B-\")\n elif gpa > 2.0:\n letter_grades.append(\"C+\")\n elif gpa > 1.7:\n letter_grades.append(\"C\")\n elif gpa > 1.3:\n letter_grades.append(\"C-\")\n elif gpa > 1.0:\n letter_grades.append(\"D+\")\n elif gpa > 0.7:\n letter_grades.append(\"D\")\n elif gpa > 0.0:\n letter_grades.append(\"D-\")\n else:\n letter_grades.append(\"E\")\n return letter_grades"} 13 | {"task_id": "HumanEval/157", "completion": "\n sides = sorted([a, b, c])\n return sides[0] ** 2 + sides[1] ** 2 == sides[2] ** 2"} 14 | {"task_id": "HumanEval/15", "completion": "\n return \" \".join(str(i) for i in range(n+1))"} 15 | {"task_id": "HumanEval/87", "completion": "\n result = []\n for i, row in enumerate(lst):\n for j, value in enumerate(row):\n if value == x:\n result.append((i, j))\n result.sort(key=lambda coord: (coord[0], -coord[1]))\n return result"} 16 | {"task_id": "HumanEval/161", "completion": "\n # Check if the string contains any letters\n contains_letters = any(c.isalpha() for c in s)\n\n # If the string contains no letters, reverse the string\n if not contains_letters:\n return s[::-1]\n\n # Otherwise, reverse the case of each letter in the string\n result = \"\"\n for c in s:\n if c.isalpha():\n result += c.lower() if c.isupper() else c.upper()\n else:\n result += c\n\n return result"} 17 | {"task_id": "HumanEval/89", "completion": "\n encrypted = \"\"\n for char in s:\n if char.isalpha():\n shift = 2 * 2\n char_code = ord(char) + shift\n if char.islower():\n encrypted += chr(char_code) if char_code <= ord('z') else chr(ord('a') + char_code - ord('z') - 1)\n else:\n encrypted += chr(char_code) if char_code <= ord('Z') else chr(ord('A') + char_code - ord('Z') - 1)\n else:\n encrypted += char\n return encrypted"} 18 | {"task_id": "HumanEval/17", "completion": "\n beats = []\n notes = music_string.split()\n for note in notes:\n if note == 'o':\n beats.append(4)\n elif note == 'o|':\n beats.append(2)\n elif note == '.|':\n beats.append(1)\n return beats"} 19 | {"task_id": "HumanEval/159", "completion": "\n # Calculate the number of carrots to eat\n carrots_to_eat = min(need, remaining)\n\n # Update the total number of eaten carrots and the remaining carrots\n total_eaten = number + carrots_to_eat\n remaining -= carrots_to_eat\n\n # Return the result as an array\n return [total_eaten, remaining]"} 20 | {"task_id": "HumanEval/163", "completion": "\n if a > b:\n a, b = b, a\n return [i for i in range(a, b + 1) if i % 2 == 0]"} 21 | {"task_id": "HumanEval/91", "completion": "\n import re\n\n # Split the string into sentences\n sentences = re.split(r'[.?!]', S)\n\n # Count the number of boredoms\n boredom_count = 0\n for sentence in sentences:\n words = sentence.strip().split()\n if words and words[0] == \"I\":\n boredom_count += 1\n\n return boredom_count"} 22 | {"task_id": "HumanEval/22", "completion": "\n return [value for value in values if isinstance(value, int)]"} 23 | {"task_id": "HumanEval/23", "completion": "\n return len(string)"} 24 | {"task_id": "HumanEval/21", "completion": "\n min_num = min(numbers)\n max_num = max(numbers)\n range_num = max_num - min_num\n return [(num - min_num) / range_num for num in numbers]"} 25 | {"task_id": "HumanEval/20", "completion": "\n numbers.sort()\n min_diff = float('inf')\n closest_pair = (0, 0)\n\n for i in range(len(numbers) - 1):\n diff = numbers[i + 1] - numbers[i]\n if diff < min_diff:\n min_diff = diff\n closest_pair = (numbers[i], numbers[i + 1])\n\n return closest_pair"} 26 | {"task_id": "HumanEval/24", "completion": "\n for i in range(n-1, 0, -1):\n if n % i == 0:\n return i"} 27 | {"task_id": "HumanEval/19", "completion": "\n # Create a dictionary to map number words to their integer values\n number_map = {\n 'zero': 0,\n 'one': 1,\n 'two': 2,\n 'three': 3,\n 'four': 4,\n 'five': 5,\n 'six': 6,\n 'seven': 7,\n 'eight': 8,\n 'nine': 9\n }\n\n # Split the input string into a list of number words\n number_words = numbers.split()\n\n # Convert the number words to their integer values using the number_map\n number_values = [number_map[word] for word in number_words]\n\n # Sort the integer values\n sorted_values = sorted(number_values)\n\n # Convert the sorted integer values back to number words using the number_map\n sorted_words = [key for value in sorted_values for key, mapped_value in number_map.items() if value == mapped_value]\n\n # Join the sorted number words into a single string and return it\n return ' '.join(sorted_words)"} 28 | {"task_id": "HumanEval/27", "completion": "\n return ''.join(char.lower() if char.isupper() else char.upper() for char in string)"} 29 | {"task_id": "HumanEval/28", "completion": "\n result = \"\".join(strings)\n return result"} 30 | {"task_id": "HumanEval/26", "completion": "\n unique_numbers = []\n for number in numbers:\n if numbers.count(number) == 1:\n unique_numbers.append(number)\n return unique_numbers"} 31 | {"task_id": "HumanEval/25", "completion": "\n factors = []\n i = 2\n while i * i <= n:\n if n % i:\n i += 1\n else:\n n //= i\n factors.append(i)\n if n > 1:\n factors.append(n)\n return factors"} 32 | {"task_id": "HumanEval/29", "completion": "\n return [string for string in strings if string.startswith(prefix)]"} 33 | {"task_id": "HumanEval/30", "completion": "\n return [num for num in l if num > 0]"} 34 | {"task_id": "HumanEval/34", "completion": "\n return sorted(set(l))"} 35 | {"task_id": "HumanEval/31", "completion": "\n if n <= 1:\n return False\n for i in range(2, int(n**0.5) + 1):\n if n % i == 0:\n return False\n return True"} 36 | {"task_id": "HumanEval/35", "completion": "\n return max(l)"} 37 | {"task_id": "HumanEval/36", "completion": "\n count = 0\n for i in range(n):\n if i % 11 == 0 or i % 13 == 0:\n count += str(i).count('7')\n return count"} 38 | {"task_id": "HumanEval/33", "completion": "\n # Extract the values at indices divisible by 3\n divisible_by_three = [l[i] for i in range(0, len(l), 3)]\n\n # Sort the extracted values\n divisible_by_three.sort()\n\n # Replace the values at indices divisible by 3 with the sorted values\n for i, value in enumerate(divisible_by_three):\n l[i * 3] = value\n\n return l"} 39 | {"task_id": "HumanEval/32", "completion": "\n if len(xs) % 2 == 0 and xs[-1] != 0:\n left = -100\n right = 100\n epsilon = 1e-6\n\n while abs(right - left) > epsilon:\n mid = (left + right) / 2\n mid_value = poly(xs, mid)\n\n if abs(mid_value) < epsilon:\n return mid\n elif mid_value > 0:\n right = mid\n else:\n left = mid\n\n return None"} 40 | {"task_id": "HumanEval/37", "completion": "\n even_values = sorted([l[i] for i in range(0, len(l), 2)])\n result = [even_values.pop(0) if i % 2 == 0 else l[i] for i in range(len(l))]\n return result"} 41 | {"task_id": "HumanEval/38", "completion": "\n # split string to groups. Each of length 3.\n groups = [s[(3 * i):min((3 * i + 3), len(s))] for i in range((len(s) + 2) // 3)]\n # reverse cycle elements in each group. Unless group has fewer elements than 3.\n groups = [(group[-1] + group[:-1]) if len(group) == 3 else group for group in groups]\n return \"\".join(groups)"} 42 | {"task_id": "HumanEval/39", "completion": "\n def is_prime(num):\n if num < 2:\n return False\n for i in range(2, int(num**0.5) + 1):\n if num % i == 0:\n return False\n return True\n\n def fib(num):\n if num <= 1:\n return num\n else:\n return fib(num - 1) + fib(num - 2)\n\n count = 0\n i = 0\n while count < n:\n fib_num = fib(i)\n if is_prime(fib_num):\n count += 1\n i += 1\n\n return fib_num"} 43 | {"task_id": "HumanEval/40", "completion": "\n n = len(l)\n if n < 3:\n return False\n\n l.sort()\n\n for i in range(n - 2):\n if i > 0 and l[i] == l[i - 1]:\n continue\n left, right = i + 1, n - 1\n while left < right:\n total = l[i] + l[left] + l[right]\n if total == 0:\n return True\n elif total < 0:\n left += 1\n else:\n right -= 1\n\n return False"} 44 | {"task_id": "HumanEval/41", "completion": "\n # Since there are n cars moving in each direction and they all move at the same speed,\n # each car moving left to right will collide with each car moving right to left.\n # Therefore, the total number of collisions is n * n.\n return n * n"} 45 | {"task_id": "HumanEval/115", "completion": "\n total_water = sum([row.count(1) for row in grid])\n return math.ceil(total_water / capacity)"} 46 | {"task_id": "HumanEval/116", "completion": "\n return sorted(arr, key=lambda x: (bin(x).count('1'), x))"} 47 | {"task_id": "HumanEval/117", "completion": "\n def count_consonants(word):\n consonants = \"bcdfghjklmnpqrstvwxyzBCDFGHJKLMNPQRSTVWXYZ\"\n count = 0\n for char in word:\n if char in consonants:\n count += 1\n return count\n\n words = s.split()\n result = []\n\n for word in words:\n if count_consonants(word) == n:\n result.append(word)\n\n return result"} 48 | {"task_id": "HumanEval/118", "completion": "\n vowels = \"AEIOUaeiou\"\n closest_vowel = \"\"\n\n for i in range(len(word) - 1, 0, -1):\n if word[i] in vowels and word[i - 1] not in vowels and word[i + 1] not in vowels:\n closest_vowel = word[i]\n break\n\n return closest_vowel"} 49 | {"task_id": "HumanEval/119", "completion": "\n def is_good(s):\n stack = []\n for c in s:\n if c == '(':\n stack.append(c)\n elif stack and stack[-1] == '(':\n stack.pop()\n else:\n return False\n return not stack\n\n if is_good(lst[0] + lst[1]) or is_good(lst[1] + lst[0]):\n return 'Yes'\n else:\n return 'No'"} 50 | {"task_id": "HumanEval/121", "completion": "\n odd_sum = 0\n for i in range(1, len(lst), 2):\n if lst[i] % 2 != 0:\n odd_sum += lst[i]\n return odd_sum"} 51 | {"task_id": "HumanEval/122", "completion": "\n result = 0\n for i in range(k):\n if 0 < arr[i] < 100:\n result += arr[i]\n return result"} 52 | {"task_id": "HumanEval/120", "completion": "\n # Sort the array in descending order\n arr.sort(reverse=True)\n\n # Return the first k elements of the sorted array\n return arr[:k]"} 53 | {"task_id": "HumanEval/123", "completion": "\n def collatz_sequence(n):\n seq = [n]\n while n != 1:\n if n % 2 == 0:\n n = n // 2\n else:\n n = 3 * n + 1\n seq.append(n)\n return seq\n\n odd_collatz = [x for x in collatz_sequence(n) if x % 2 != 0]\n return sorted(odd_collatz)"} 54 | {"task_id": "HumanEval/125", "completion": "\n if ' ' in txt:\n return txt.split()\n elif ',' in txt:\n return txt.split(',')\n else:\n odd_count = 0\n for char in txt:\n if char.islower() and (ord(char) - ord('a')) % 2 == 1:\n odd_count += 1\n return odd_count"} 55 | {"task_id": "HumanEval/126", "completion": "\n for i in range(len(lst) - 1):\n if lst[i] > lst[i + 1]:\n return False\n if lst.count(lst[i]) > 2:\n return False\n return True"} 56 | {"task_id": "HumanEval/128", "completion": "\n if not arr:\n return None\n\n sum_magnitudes = 0\n prod_signs = 1\n\n for num in arr:\n if num == 0:\n return 0\n elif num < 0:\n prod_signs *= -1\n sum_magnitudes += abs(num)\n else:\n sum_magnitudes += num\n\n return sum_magnitudes * prod_signs"} 57 | {"task_id": "HumanEval/124", "completion": "\n import re\n\n # Check if the date string is empty\n if not date:\n return False\n\n # Check if the date string is in the correct format\n pattern = re.compile(r'^\\d{2}-\\d{2}-\\d{4}$')\n if not pattern.match(date):\n return False\n\n # Split the date string into month, day, and year\n month, day, year = map(int, date.split('-'))\n\n # Check if the month is valid\n if month < 1 or month > 12:\n return False\n\n # Check if the day is valid\n if month in [1, 3, 5, 7, 8, 10, 12]:\n if day < 1 or day > 31:\n return False\n elif month in [4, 6, 9, 11]:\n if day < 1 or day > 30:\n return False\n else: # month == 2\n if day < 1 or day > 29:\n return False\n\n return True"} 58 | {"task_id": "HumanEval/127", "completion": "\n def is_prime(n):\n if n < 2:\n return False\n for i in range(2, int(n**0.5) + 1):\n if n % i == 0:\n return False\n return True\n\n start = max(interval1[0], interval2[0])\n end = min(interval1[1], interval2[1])\n\n if start > end:\n return \"NO\"\n\n intersection_length = end - start\n\n if is_prime(intersection_length):\n return \"YES\"\n else:\n return \"NO\""} 59 | {"task_id": "HumanEval/131", "completion": "\n product = 1\n has_odd_digit = False\n\n for digit in str(n):\n if int(digit) % 2 != 0:\n product *= int(digit)\n has_odd_digit = True\n\n return product if has_odd_digit else 0"} 60 | {"task_id": "HumanEval/132", "completion": "\n stack = []\n nested = False\n\n for char in string:\n if char == '[':\n stack.append(char)\n elif char == ']':\n if not stack:\n return False\n else:\n if stack[-1] == '[':\n nested = True\n stack.pop()\n\n return nested and not stack"} 61 | {"task_id": "HumanEval/130", "completion": "\n def tribonacci(n, memo={}):\n if n == 1:\n return 3\n if n % 2 == 0:\n return 1 + n // 2\n if n in memo:\n return memo[n]\n memo[n] = tribonacci(n - 1) + tribonacci(n - 2) + tribonacci(n + 1)\n return memo[n]\n\n return [tribonacci(i) for i in range(n + 1)]"} 62 | {"task_id": "HumanEval/134", "completion": "\n if not txt or txt[-1] == \" \":\n return False\n return txt[-1].isalpha() and txt[-2] == \" \""} 63 | {"task_id": "HumanEval/133", "completion": "\n import math\n\n # Round each element in the list to the upper int (Ceiling)\n rounded_lst = [math.ceil(num) for num in lst]\n\n # Calculate the sum of squared numbers in the rounded list\n sum_of_squares = sum([num ** 2 for num in rounded_lst])\n\n return sum_of_squares"} 64 | {"task_id": "HumanEval/135", "completion": "\n largest_index = -1\n for i in range(1, len(arr)):\n if arr[i] < arr[i - 1]:\n largest_index = i\n return largest_index"} 65 | {"task_id": "HumanEval/138", "completion": "\n if n < 8:\n return False\n return (n - 8) % 4 == 0"} 66 | {"task_id": "HumanEval/129", "completion": "\n def dfs(i, j, k, path, visited):\n if k == 0:\n return path\n visited.add((i, j))\n min_path = None\n for x, y in [(i-1, j), (i+1, j), (i, j-1), (i, j+1)]:\n if 0 <= x < len(grid) and 0 <= y < len(grid[0]) and (x, y) not in visited:\n new_path = dfs(x, y, k-1, path + [grid[x][y]], visited.copy())\n if min_path is None or new_path < min_path:\n min_path = new_path\n return min_path\n\n min_path = None\n for i in range(len(grid)):\n for j in range(len(grid[0])):\n path = dfs(i, j, k-1, [grid[i][j]], set())\n if min_path is None or path < min_path:\n min_path = path\n\n return min_path"} 67 | {"task_id": "HumanEval/136", "completion": "\n largest_negative = None\n smallest_positive = None\n\n for num in lst:\n if num < 0:\n if largest_negative is None or num > largest_negative:\n largest_negative = num\n elif num > 0:\n if smallest_positive is None or num < smallest_positive:\n smallest_positive = num\n\n return (largest_negative, smallest_positive)"} 68 | {"task_id": "HumanEval/139", "completion": "\n def factorial(x):\n if x == 0:\n return 1\n else:\n return x * factorial(x - 1)\n\n result = 1\n for i in range(1, n + 1):\n result *= factorial(i)\n\n return result"} 69 | {"task_id": "HumanEval/137", "completion": "\n def convert_to_float(value):\n if isinstance(value, str):\n value = value.replace(\",\", \".\")\n return float(value)\n return value\n\n a_float = convert_to_float(a)\n b_float = convert_to_float(b)\n\n if a_float == b_float:\n return None\n elif a_float > b_float:\n return a\n else:\n return b"} 70 | {"task_id": "HumanEval/140", "completion": "\n import re\n\n # Replace more than 2 consecutive spaces with -\n text = re.sub(r\" {3,}\", \"-\", text)\n\n # Replace remaining spaces with underscores\n text = text.replace(\" \", \"_\")\n\n return text"} 71 | {"task_id": "HumanEval/142", "completion": "\n result = 0\n for i, num in enumerate(lst):\n if i % 3 == 0 and i % 4 != 0:\n result += num ** 2\n elif i % 4 == 0:\n result += num ** 3\n else:\n result += num\n return result"} 72 | {"task_id": "HumanEval/143", "completion": "\n def is_prime(num):\n if num < 2:\n return False\n for i in range(2, num):\n if num % i == 0:\n return False\n return True\n\n words = sentence.split()\n prime_words = [word for word in words if is_prime(len(word))]\n return ' '.join(prime_words)"} 73 | {"task_id": "HumanEval/144", "completion": "\n from fractions import Fraction\n\n # Convert the strings to Fraction objects\n x_frac = Fraction(x)\n n_frac = Fraction(n)\n\n # Multiply the fractions\n result = x_frac * n_frac\n\n # Check if the result is a whole number\n return result.denominator == 1"} 74 | {"task_id": "HumanEval/145", "completion": "\n def sum_of_digits(num):\n return sum(int(digit) for digit in str(abs(num)))\n\n return sorted(nums, key=lambda x: (sum_of_digits(x), nums.index(x)))"} 75 | {"task_id": "HumanEval/141", "completion": "\n import re\n\n # Check if the file's name contains exactly one dot\n if file_name.count('.') != 1:\n return 'No'\n\n # Split the file's name into name and extension\n name, extension = file_name.split('.')\n\n # Check if the substring before the dot is not empty and starts with a latin alphabet letter\n if not name or not re.match(r'^[a-zA-Z]', name):\n return 'No'\n\n # Check if the substring after the dot is one of the allowed extensions\n if extension not in ['txt', 'exe', 'dll']:\n return 'No'\n\n # Check if there are not more than three digits in the file's name\n if len(re.findall(r'\\d', name)) > 3:\n return 'No'\n\n return 'Yes'"} 76 | {"task_id": "HumanEval/1", "completion": "\n result = []\n stack = []\n start = 0\n\n for i, char in enumerate(paren_string):\n if char == '(':\n if not stack:\n start = i\n stack.append(char)\n elif char == ')':\n stack.pop()\n if not stack:\n result.append(paren_string[start:i+1])\n\n return result"} 77 | {"task_id": "HumanEval/146", "completion": "\n def is_odd_digit(num):\n return num in [1, 3, 5, 7, 9]\n\n def first_and_last_odd(num):\n num_str = str(abs(num))\n return is_odd_digit(int(num_str[0])) and is_odd_digit(int(num_str[-1]))\n\n count = 0\n for num in nums:\n if num > 10 and first_and_last_odd(num):\n count += 1\n\n return count"} 78 | {"task_id": "HumanEval/3", "completion": "\n balance = 0\n for operation in operations:\n balance += operation\n if balance < 0:\n return True\n return False"} 79 | {"task_id": "HumanEval/74", "completion": "\n # Calculate the total number of characters in each list\n total_chars_lst1 = sum(len(s) for s in lst1)\n total_chars_lst2 = sum(len(s) for s in lst2)\n\n # Compare the total number of characters and return the list with fewer characters\n if total_chars_lst1 <= total_chars_lst2:\n return lst1\n else:\n return lst2"} 80 | {"task_id": "HumanEval/5", "completion": "\n result = []\n for i, num in enumerate(numbers):\n if i > 0:\n result.append(delimeter)\n result.append(num)\n return result"} 81 | {"task_id": "HumanEval/150", "completion": "\n def is_prime(num):\n if num < 2:\n return False\n for i in range(2, num):\n if num % i == 0:\n return False\n return True\n\n if is_prime(n):\n return x\n else:\n return y"} 82 | {"task_id": "HumanEval/76", "completion": "\n if n == 1:\n return x == 1\n i = 0\n while n ** i <= x:\n if n ** i == x:\n return True\n i += 1\n return False"} 83 | {"task_id": "HumanEval/148", "completion": "\n planets = {\n \"Mercury\": 1,\n \"Venus\": 2,\n \"Earth\": 3,\n \"Mars\": 4,\n \"Jupiter\": 5,\n \"Saturn\": 6,\n \"Uranus\": 7,\n \"Neptune\": 8\n }\n\n if planet1 not in planets or planet2 not in planets:\n return ()\n\n planet1_index = planets[planet1]\n planet2_index = planets[planet2]\n\n if planet1_index > planet2_index:\n planet1_index, planet2_index = planet2_index, planet1_index\n\n result = [planet for planet, index in planets.items() if planet1_index < index < planet2_index]\n\n return tuple(result)"} 84 | {"task_id": "HumanEval/154", "completion": "\n for i in range(len(b)):\n rotated_b = b[i:] + b[:i]\n if rotated_b in a:\n return True\n return False"} 85 | {"task_id": "HumanEval/84", "completion": "\n # Convert N to binary and count the number of 1s\n binary_sum = bin(N).count('1')\n\n # Convert the binary_sum to binary string and return it\n return bin(binary_sum)[2:]"} 86 | {"task_id": "HumanEval/12", "completion": "\n if not strings:\n return None\n\n longest_string = strings[0]\n for string in strings:\n if len(string) > len(longest_string):\n longest_string = string\n\n return longest_string"} 87 | {"task_id": "HumanEval/79", "completion": "\n binary = bin(decimal)[2:] # Convert decimal to binary and remove the '0b' prefix\n return f'db{binary}db' # Add 'db' at the beginning and end of the binary string"} 88 | {"task_id": "HumanEval/16", "completion": "\n # Convert the string to lowercase\n string = string.lower()\n\n # Create a set to store distinct characters\n distinct_characters = set()\n\n # Iterate through the string and add each character to the set\n for char in string:\n distinct_characters.add(char)\n\n # Return the length of the set, which represents the number of distinct characters\n return len(distinct_characters)"} 89 | {"task_id": "HumanEval/162", "completion": "\n import hashlib\n\n if not text:\n return None\n\n md5_hash = hashlib.md5()\n md5_hash.update(text.encode('utf-8'))\n return md5_hash.hexdigest()"} 90 | {"task_id": "HumanEval/158", "completion": "\n max_unique_chars = 0\n max_word = \"\"\n\n for word in words:\n unique_chars = len(set(word))\n if unique_chars > max_unique_chars:\n max_unique_chars = unique_chars\n max_word = word\n elif unique_chars == max_unique_chars:\n if word < max_word:\n max_word = word\n\n return max_word"} 91 | {"task_id": "HumanEval/78", "completion": "\n prime_hex_digits = {'2', '3', '5', '7', 'B', 'D'}\n count = 0\n\n for digit in num:\n if digit in prime_hex_digits:\n count += 1\n\n return count"} 92 | {"task_id": "HumanEval/98", "completion": "\n uppercase_vowels = \"AEIOU\"\n count = 0\n\n for i in range(0, len(s), 2):\n if s[i] in uppercase_vowels:\n count += 1\n\n return count"} 93 | {"task_id": "HumanEval/88", "completion": "\n if not array:\n return array\n\n sorted_array = array.copy()\n first_last_sum = array[0] + array[-1]\n\n if first_last_sum % 2 == 0:\n sorted_array.sort(reverse=True)\n else:\n sorted_array.sort()\n\n return sorted_array"} 94 | {"task_id": "HumanEval/92", "completion": "\n if isinstance(x, int) and isinstance(y, int) and isinstance(z, int):\n return (x == y + z) or (y == x + z) or (z == x + y)\n else:\n return False"} 95 | {"task_id": "HumanEval/96", "completion": "\n def is_prime(num):\n if num < 2:\n return False\n for i in range(2, num):\n if num % i == 0:\n return False\n return True\n\n primes = []\n for i in range(n):\n if is_prime(i):\n primes.append(i)\n return primes"} 96 | {"task_id": "HumanEval/102", "completion": "\n if x > y:\n return -1\n if y % 2 == 0:\n return y\n elif y - 1 >= x:\n return y - 1\n else:\n return -1"} 97 | {"task_id": "HumanEval/104", "completion": "\n def has_even_digit(num):\n for digit in str(num):\n if int(digit) % 2 == 0:\n return True\n return False\n\n result = [num for num in x if not has_even_digit(num)]\n return sorted(result)"} 98 | {"task_id": "HumanEval/100", "completion": "\n stones = [n]\n for i in range(1, n):\n if n % 2 == 0:\n stones.append(stones[-1] + 2)\n else:\n stones.append(stones[-1] + 2)\n return stones"} 99 | {"task_id": "HumanEval/94", "completion": "\n def is_prime(n):\n if n < 2:\n return False\n for i in range(2, int(n**0.5) + 1):\n if n % i == 0:\n return False\n return True\n\n def sum_of_digits(n):\n return sum(int(digit) for digit in str(n))\n\n largest_prime = max(filter(is_prime, lst), default=0)\n return sum_of_digits(largest_prime)"} 100 | {"task_id": "HumanEval/0", "completion": "\n for i in range(len(numbers)):\n for j in range(i + 1, len(numbers)):\n if abs(numbers[i] - numbers[j]) < threshold:\n return True\n return False"} 101 | {"task_id": "HumanEval/106", "completion": "\n def factorial(i):\n if i == 0 or i == 1:\n return 1\n else:\n return i * factorial(i - 1)\n\n def sum_of_numbers(i):\n return sum(range(1, i + 1))\n\n result = []\n for i in range(1, n + 1):\n if i % 2 == 0:\n result.append(factorial(i))\n else:\n result.append(sum_of_numbers(i))\n\n return result"} 102 | {"task_id": "HumanEval/112", "completion": "\n # Remove characters in c from s\n result = ''.join([char for char in s if char not in c])\n\n # Check if the result is a palindrome\n is_palindrome = result == result[::-1]\n\n return (result, is_palindrome)"} 103 | {"task_id": "HumanEval/110", "completion": "\n odd_count_lst1 = sum(1 for num in lst1 if num % 2 != 0)\n even_count_lst2 = sum(1 for num in lst2 if num % 2 == 0)\n\n if odd_count_lst1 <= even_count_lst2:\n return \"YES\"\n else:\n return \"NO\""} 104 | {"task_id": "HumanEval/108", "completion": "\n def sum_of_digits(num):\n \"\"\"\n Helper function to calculate the sum of signed digits of a number.\n \"\"\"\n if num < 0:\n return -1 * int(str(num)[1]) + sum(int(digit) for digit in str(num)[2:])\n else:\n return sum(int(digit) for digit in str(num))\n\n count = 0\n for num in arr:\n if sum_of_digits(num) > 0:\n count += 1\n\n return count"} 105 | {"task_id": "HumanEval/42", "completion": "\n return [x + 1 for x in l]"} 106 | {"task_id": "HumanEval/44", "completion": "\n if x == 0:\n return '0'\n digits = []\n while x:\n digits.append(str(x % base))\n x //= base\n return ''.join(digits[::-1])"} 107 | {"task_id": "HumanEval/50", "completion": "\n return \"\".join([chr(((ord(ch) - 5 - ord(\"a\")) % 26) + ord(\"a\")) for ch in s])"} 108 | {"task_id": "HumanEval/48", "completion": "\n # Remove any non-alphanumeric characters and convert to lowercase\n cleaned_text = ''.join(char.lower() for char in text if char.isalnum())\n \n # Check if the cleaned text is equal to its reverse\n return cleaned_text == cleaned_text[::-1]"} 109 | {"task_id": "HumanEval/52", "completion": "\n return all(x < t for x in l)"} 110 | {"task_id": "HumanEval/54", "completion": "\n # Create sets of unique characters from both strings\n set_s0 = set(s0)\n set_s1 = set(s1)\n\n # Compare the sets to see if they have the same characters\n return set_s0 == set_s1"} 111 | {"task_id": "HumanEval/60", "completion": "\n return sum(range(1, n+1))"} 112 | {"task_id": "HumanEval/58", "completion": "\n common_elements = set(l1) & set(l2)\n return sorted(list(common_elements))"} 113 | {"task_id": "HumanEval/56", "completion": "\n count = 0\n for bracket in brackets:\n if bracket == \"<\":\n count += 1\n elif bracket == \">\":\n count -= 1\n if count < 0:\n return False\n return count == 0"} 114 | {"task_id": "HumanEval/46", "completion": "\n if n == 0:\n return 0\n elif n == 1:\n return 0\n elif n == 2:\n return 2\n elif n == 3:\n return 0\n\n a, b, c, d = 0, 0, 2, 0\n for _ in range(n - 3):\n a, b, c, d = b, c, d, a + b + c + d\n\n return d"} 115 | {"task_id": "HumanEval/62", "completion": "\n result = []\n for i in range(1, len(xs)):\n result.append(i * xs[i])\n return result"} 116 | {"task_id": "HumanEval/66", "completion": "\n upper_sum = 0\n for char in s:\n if char.isupper():\n upper_sum += ord(char)\n return upper_sum"} 117 | {"task_id": "HumanEval/64", "completion": "\n s = s.lower()\n count = 0\n vowels = \"aeiou\"\n for char in s:\n if char in vowels:\n count += 1\n if s[-1] == \"y\":\n count += 1\n return count"} 118 | {"task_id": "HumanEval/70", "completion": "\n sorted_lst = sorted(lst)\n result = []\n while sorted_lst:\n result.append(sorted_lst.pop(0))\n if sorted_lst:\n result.append(sorted_lst.pop(-1))\n return result"} 119 | {"task_id": "HumanEval/2", "completion": "\n decimal_part = number - int(number)\n return decimal_part"} 120 | {"task_id": "HumanEval/68", "completion": "\n smallest_even = None\n smallest_index = None\n\n for i, value in enumerate(arr):\n if value % 2 == 0:\n if smallest_even is None or value < smallest_even:\n smallest_even = value\n smallest_index = i\n\n if smallest_even is not None:\n return [smallest_even, smallest_index]\n else:\n return []"} 121 | {"task_id": "HumanEval/75", "completion": "\n def is_prime(n):\n if n < 2:\n return False\n for i in range(2, n):\n if n % i == 0:\n return False\n return True\n\n prime_factors = []\n for i in range(2, a):\n if a % i == 0 and is_prime(i):\n prime_factors.append(i)\n a = a // i\n if len(prime_factors) == 3 and a == 1:\n return True\n elif len(prime_factors) > 3:\n return False\n return False"} 122 | {"task_id": "HumanEval/72", "completion": "\n # Check if the list is palindromic\n is_balanced = q == q[::-1]\n\n # Calculate the sum of the elements in the list\n total_weight = sum(q)\n\n # Return True if the list is balanced and the total weight is less than or equal to the maximum possible weight\n return is_balanced and total_weight <= w"} 123 | {"task_id": "HumanEval/6", "completion": "\n def max_nesting(s: str) -> int:\n count = 0\n max_count = 0\n for char in s:\n if char == '(':\n count += 1\n max_count = max(max_count, count)\n elif char == ')':\n count -= 1\n return max_count\n\n groups = paren_string.split()\n return [max_nesting(group) for group in groups]"} 124 | {"task_id": "HumanEval/149", "completion": "\n # Remove strings with odd lengths\n even_length_strings = [s for s in lst if len(s) % 2 == 0]\n\n # Sort the list by length and then alphabetically\n sorted_list = sorted(even_length_strings, key=lambda x: (len(x), x))\n\n return sorted_list"} 125 | {"task_id": "HumanEval/14", "completion": "\n return [string[:i+1] for i in range(len(string))]"} 126 | {"task_id": "HumanEval/82", "completion": "\n def is_prime(n):\n if n < 2:\n return False\n for i in range(2, int(n**0.5) + 1):\n if n % i == 0:\n return False\n return True\n\n return is_prime(len(string))"} 127 | {"task_id": "HumanEval/90", "completion": "\n if len(lst) < 2:\n return None\n unique_lst = list(set(lst))\n unique_lst.sort()\n if len(unique_lst) < 2:\n return None\n return unique_lst[1]"} 128 | {"task_id": "HumanEval/97", "completion": "\n # Get the unit digits of a and b\n unit_digit_a = abs(a) % 10\n unit_digit_b = abs(b) % 10\n\n # Return the product of the unit digits\n return unit_digit_a * unit_digit_b"} 129 | {"task_id": "HumanEval/101", "completion": "\n # Replace commas with spaces\n s = s.replace(\",\", \" \")\n \n # Split the string into words\n words = s.split()\n \n # Return the list of words\n return words"} 130 | {"task_id": "HumanEval/156", "completion": "\n val = [\n 1000, 900, 500, 400,\n 100, 90, 50, 40,\n 10, 9, 5, 4,\n 1\n ]\n syb = [\n \"M\", \"CM\", \"D\", \"CD\",\n \"C\", \"XC\", \"L\", \"XL\",\n \"X\", \"IX\", \"V\", \"IV\",\n \"I\"\n ]\n roman_num = ''\n i = 0\n while number > 0:\n for _ in range(number // val[i]):\n roman_num += syb[i]\n number -= val[i]\n i += 1\n return roman_num.lower()"} 131 | {"task_id": "HumanEval/93", "completion": "\n vowels = \"aeiouAEIOU\"\n encoded_message = \"\"\n\n for char in message:\n if char.isalpha():\n if char in vowels:\n if char.islower():\n encoded_message += chr(((ord(char) - ord('a') + 2) % 26) + ord('a'))\n else:\n encoded_message += chr(((ord(char) - ord('A') + 2) % 26) + ord('A'))\n else:\n encoded_message += char.swapcase()\n else:\n encoded_message += char\n\n return encoded_message"} 132 | {"task_id": "HumanEval/47", "completion": "\n l.sort()\n n = len(l)\n if n % 2 == 0:\n return (l[n//2 - 1] + l[n//2]) / 2\n else:\n return l[n//2]"} 133 | {"task_id": "HumanEval/113", "completion": "\n result = []\n for s in lst:\n odd_count = sum(1 for c in s if int(c) % 2 == 1)\n result.append(f\"the number of odd elements {odd_count}n the str{odd_count}ng {odd_count} of the {odd_count}nput.\")\n return result"} 134 | {"task_id": "HumanEval/43", "completion": "\n for i in range(len(l)):\n for j in range(i+1, len(l)):\n if l[i] + l[j] == 0:\n return True\n return False"} 135 | {"task_id": "HumanEval/55", "completion": "\n if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fib(n-1) + fib(n-2)"} 136 | -------------------------------------------------------------------------------- /human_eval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getcursor/eval/20f1ae654a1a75d7c55f324f8b5b6a7f88b34953/human_eval/__init__.py -------------------------------------------------------------------------------- /human_eval/data.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable, Dict 2 | import gzip 3 | import json 4 | import os 5 | 6 | 7 | ROOT = os.path.dirname(os.path.abspath(__file__)) 8 | HUMAN_EVAL = os.path.join(ROOT, "..", "data", "HumanEval.jsonl.gz") 9 | 10 | 11 | def read_problems(evalset_file: str = HUMAN_EVAL) -> Dict[str, Dict]: 12 | return {task["task_id"]: task for task in stream_jsonl(evalset_file)} 13 | 14 | 15 | def stream_jsonl(filename: str) -> Iterable[Dict]: 16 | """ 17 | Parses each jsonl line and yields it as a dictionary 18 | """ 19 | if filename.endswith(".gz"): 20 | with open(filename, "rb") as gzfp: 21 | with gzip.open(gzfp, 'rt') as fp: 22 | for line in fp: 23 | if any(not x.isspace() for x in line): 24 | yield json.loads(line) 25 | else: 26 | with open(filename, "r") as fp: 27 | for line in fp: 28 | if any(not x.isspace() for x in line): 29 | yield json.loads(line) 30 | 31 | 32 | def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False): 33 | """ 34 | Writes an iterable of dictionaries to jsonl 35 | """ 36 | if append: 37 | mode = 'ab' 38 | else: 39 | mode = 'wb' 40 | filename = os.path.expanduser(filename) 41 | if filename.endswith(".gz"): 42 | with open(filename, mode) as fp: 43 | with gzip.GzipFile(fileobj=fp, mode='wb') as gzfp: 44 | for x in data: 45 | gzfp.write((json.dumps(x) + "\n").encode('utf-8')) 46 | else: 47 | with open(filename, mode) as fp: 48 | for x in data: 49 | fp.write((json.dumps(x) + "\n").encode('utf-8')) 50 | -------------------------------------------------------------------------------- /human_eval/evaluate_functional_correctness.py: -------------------------------------------------------------------------------- 1 | import fire 2 | import sys 3 | 4 | from human_eval.data import HUMAN_EVAL 5 | from human_eval.evaluation import evaluate_functional_correctness 6 | 7 | 8 | def entry_point( 9 | sample_file: str, 10 | k: str = "1,10,100", 11 | n_workers: int = 4, 12 | timeout: float = 3.0, 13 | problem_file: str = HUMAN_EVAL, 14 | ): 15 | """ 16 | Evaluates the functional correctness of generated samples, and writes 17 | results to f"{sample_file}_results.jsonl.gz" 18 | """ 19 | k = list(map(int, k.split(","))) 20 | results = evaluate_functional_correctness(sample_file, k, n_workers, timeout, problem_file) 21 | print(results) 22 | 23 | 24 | def main(): 25 | fire.Fire(entry_point) 26 | 27 | 28 | sys.exit(main()) 29 | -------------------------------------------------------------------------------- /human_eval/evaluation.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict, Counter 2 | from concurrent.futures import ThreadPoolExecutor, as_completed 3 | from typing import List, Union, Iterable, Dict 4 | import itertools 5 | 6 | import numpy as np 7 | import tqdm 8 | 9 | from human_eval.data import HUMAN_EVAL, read_problems, stream_jsonl, write_jsonl 10 | from human_eval.execution import check_correctness 11 | 12 | 13 | def estimate_pass_at_k( 14 | num_samples: Union[int, List[int], np.ndarray], 15 | num_correct: Union[List[int], np.ndarray], 16 | k: int 17 | ) -> np.ndarray: 18 | """ 19 | Estimates pass@k of each problem and returns them in an array. 20 | """ 21 | 22 | def estimator(n: int, c: int, k: int) -> float: 23 | """ 24 | Calculates 1 - comb(n - c, k) / comb(n, k). 25 | """ 26 | if n - c < k: 27 | return 1.0 28 | return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) 29 | 30 | if isinstance(num_samples, int): 31 | num_samples_it = itertools.repeat(num_samples, len(num_correct)) 32 | else: 33 | assert len(num_samples) == len(num_correct) 34 | num_samples_it = iter(num_samples) 35 | 36 | return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]) 37 | 38 | 39 | def evaluate_functional_correctness( 40 | sample_file: str, 41 | k: List[int] = [1, 10, 100], 42 | n_workers: int = 4, 43 | timeout: float = 3.0, 44 | problem_file: str = HUMAN_EVAL, 45 | ): 46 | """ 47 | Evaluates the functional correctness of generated samples, and writes 48 | results to f"{sample_file}_results.jsonl.gz" 49 | """ 50 | 51 | problems = read_problems(problem_file) 52 | 53 | # Check the generated samples against test suites. 54 | with ThreadPoolExecutor(max_workers=n_workers) as executor: 55 | 56 | futures = [] 57 | completion_id = Counter() 58 | n_samples = 0 59 | results = defaultdict(list) 60 | 61 | print("Reading samples...") 62 | for sample in tqdm.tqdm(stream_jsonl(sample_file)): 63 | task_id = sample["task_id"] 64 | completion = sample["completion"] 65 | args = (problems[task_id], completion, timeout, completion_id[task_id]) 66 | future = executor.submit(check_correctness, *args) 67 | futures.append(future) 68 | completion_id[task_id] += 1 69 | n_samples += 1 70 | 71 | assert len(completion_id) == len(problems), "Some problems are not attempted." 72 | 73 | print("Running test suites...") 74 | for future in tqdm.tqdm(as_completed(futures), total=len(futures)): 75 | result = future.result() 76 | results[result["task_id"]].append((result["completion_id"], result)) 77 | 78 | # Calculate pass@k. 79 | total, correct = [], [] 80 | for result in results.values(): 81 | result.sort() 82 | passed = [r[1]["passed"] for r in result] 83 | total.append(len(passed)) 84 | correct.append(sum(passed)) 85 | total = np.array(total) 86 | correct = np.array(correct) 87 | 88 | ks = k 89 | pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, correct, k).mean() 90 | for k in ks if (total >= k).all()} 91 | 92 | # Finally, save the results in one file: 93 | def combine_results(): 94 | for sample in stream_jsonl(sample_file): 95 | task_id = sample["task_id"] 96 | result = results[task_id].pop(0) 97 | sample["result"] = result[1]["result"] 98 | sample["passed"] = result[1]["passed"] 99 | yield sample 100 | 101 | out_file = sample_file + "_results.jsonl" 102 | print(f"Writing results to {out_file}...") 103 | write_jsonl(out_file, tqdm.tqdm(combine_results(), total=n_samples)) 104 | 105 | return pass_at_k 106 | -------------------------------------------------------------------------------- /human_eval/execution.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Callable, Dict 2 | import ast 3 | import contextlib 4 | import faulthandler 5 | import io 6 | import os 7 | import multiprocessing 8 | import platform 9 | import signal 10 | import tempfile 11 | 12 | 13 | def unsafe_execute(problem: Dict, completion: str, timeout: float, result) -> Dict: 14 | with create_tempdir(): 15 | # These system calls are needed when cleaning up tempdir. 16 | import os 17 | import shutil 18 | rmtree = shutil.rmtree 19 | rmdir = os.rmdir 20 | chdir = os.chdir 21 | 22 | # Disable functionalities that can make destructive changes to the test. 23 | reliability_guard() 24 | 25 | # Construct the check program and run it. 26 | check_program = ( 27 | problem["prompt"] + completion + "\n" + 28 | problem["test"] + "\n" + 29 | f"check({problem['entry_point']})" 30 | ) 31 | 32 | try: 33 | exec_globals = {} 34 | with swallow_io(): 35 | with time_limit(timeout): 36 | exec(check_program, exec_globals) 37 | result.append("passed") 38 | except TimeoutException: 39 | result.append("timed out") 40 | except BaseException as e: 41 | result.append(f"failed: {e}") 42 | 43 | # Needed for cleaning up. 44 | shutil.rmtree = rmtree 45 | os.rmdir = rmdir 46 | os.chdir = chdir 47 | 48 | def check_correctness(problem: Dict, completion: str, timeout: float, 49 | completion_id: Optional[int] = None) -> Dict: 50 | """ 51 | Evaluates the functional correctness of a completion by running the test 52 | suite provided in the problem. 53 | 54 | :param completion_id: an optional completion ID so we can match 55 | the results later even if execution finishes asynchronously. 56 | """ 57 | 58 | 59 | manager = multiprocessing.Manager() 60 | result = manager.list() 61 | 62 | p = multiprocessing.Process(target=unsafe_execute, args=(problem, completion, timeout, result)) 63 | p.start() 64 | p.join(timeout=timeout + 1) 65 | if p.is_alive(): 66 | p.kill() 67 | 68 | if not result: 69 | result.append("timed out") 70 | 71 | return dict( 72 | task_id=problem["task_id"], 73 | passed=result[0] == "passed", 74 | result=result[0], 75 | completion_id=completion_id, 76 | ) 77 | 78 | 79 | @contextlib.contextmanager 80 | def time_limit(seconds: float): 81 | def signal_handler(signum, frame): 82 | raise TimeoutException("Timed out!") 83 | signal.setitimer(signal.ITIMER_REAL, seconds) 84 | signal.signal(signal.SIGALRM, signal_handler) 85 | try: 86 | yield 87 | finally: 88 | signal.setitimer(signal.ITIMER_REAL, 0) 89 | 90 | 91 | @contextlib.contextmanager 92 | def swallow_io(): 93 | stream = WriteOnlyStringIO() 94 | with contextlib.redirect_stdout(stream): 95 | with contextlib.redirect_stderr(stream): 96 | with redirect_stdin(stream): 97 | yield 98 | 99 | 100 | @contextlib.contextmanager 101 | def create_tempdir(): 102 | with tempfile.TemporaryDirectory() as dirname: 103 | with chdir(dirname): 104 | yield dirname 105 | 106 | 107 | class TimeoutException(Exception): 108 | pass 109 | 110 | 111 | class WriteOnlyStringIO(io.StringIO): 112 | """ StringIO that throws an exception when it's read from """ 113 | 114 | def read(self, *args, **kwargs): 115 | raise IOError 116 | 117 | def readline(self, *args, **kwargs): 118 | raise IOError 119 | 120 | def readlines(self, *args, **kwargs): 121 | raise IOError 122 | 123 | def readable(self, *args, **kwargs): 124 | """ Returns True if the IO object can be read. """ 125 | return False 126 | 127 | 128 | class redirect_stdin(contextlib._RedirectStream): # type: ignore 129 | _stream = 'stdin' 130 | 131 | 132 | @contextlib.contextmanager 133 | def chdir(root): 134 | if root == ".": 135 | yield 136 | return 137 | cwd = os.getcwd() 138 | os.chdir(root) 139 | try: 140 | yield 141 | except BaseException as exc: 142 | raise exc 143 | finally: 144 | os.chdir(cwd) 145 | 146 | 147 | def reliability_guard(maximum_memory_bytes: Optional[int] = None): 148 | """ 149 | This disables various destructive functions and prevents the generated code 150 | from interfering with the test (e.g. fork bomb, killing other processes, 151 | removing filesystem files, etc.) 152 | 153 | WARNING 154 | This function is NOT a security sandbox. Untrusted code, including, model- 155 | generated code, should not be blindly executed outside of one. See the 156 | Codex paper for more information about OpenAI's code sandbox, and proceed 157 | with caution. 158 | """ 159 | 160 | if maximum_memory_bytes is not None: 161 | import resource 162 | resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) 163 | resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) 164 | if not platform.uname().system == 'Darwin': 165 | resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) 166 | 167 | faulthandler.disable() 168 | 169 | import builtins 170 | builtins.exit = None 171 | builtins.quit = None 172 | 173 | import os 174 | os.environ['OMP_NUM_THREADS'] = '1' 175 | 176 | os.kill = None 177 | os.system = None 178 | os.putenv = None 179 | os.remove = None 180 | os.removedirs = None 181 | os.rmdir = None 182 | os.fchdir = None 183 | os.setuid = None 184 | os.fork = None 185 | os.forkpty = None 186 | os.killpg = None 187 | os.rename = None 188 | os.renames = None 189 | os.truncate = None 190 | os.replace = None 191 | os.unlink = None 192 | os.fchmod = None 193 | os.fchown = None 194 | os.chmod = None 195 | os.chown = None 196 | os.chroot = None 197 | os.fchdir = None 198 | os.lchflags = None 199 | os.lchmod = None 200 | os.lchown = None 201 | os.getcwd = None 202 | os.chdir = None 203 | 204 | import shutil 205 | shutil.rmtree = None 206 | shutil.move = None 207 | shutil.chown = None 208 | 209 | import subprocess 210 | subprocess.Popen = None # type: ignore 211 | 212 | __builtins__['help'] = None 213 | 214 | import sys 215 | sys.modules['ipdb'] = None 216 | sys.modules['joblib'] = None 217 | sys.modules['resource'] = None 218 | sys.modules['psutil'] = None 219 | sys.modules['tkinter'] = None 220 | -------------------------------------------------------------------------------- /prompt_utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | def parse_prompt(prompt): 4 | # Grab all blocks between the <|im_start|> and <|im_end|> tokens 5 | im_blocks = re.findall(r"<\|im_start\|>(.+?)<\|im_end\|>", prompt, re.DOTALL) 6 | 7 | messages = [] 8 | for block in im_blocks: 9 | # Separate by <|im_sep|> token 10 | role, content = block.split("<|im_sep|>") 11 | messages.append({"role": role.strip(), "content": content.strip()}) 12 | 13 | return messages 14 | 15 | def to_prompt(messages): 16 | prompt = "" 17 | for message in messages: 18 | prompt += f"<|im_start|>{message['role']}<|im_sep|>{message['content']}<|im_end|>" 19 | 20 | return prompt 21 | -------------------------------------------------------------------------------- /prompts.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | def gpt4_prompt(func: str): 4 | return [ 5 | { 6 | 'role': 'system', 7 | 'content': f''' 8 | You may only respond with code and comments and the <|start_of_completion|> token.''' 9 | }, 10 | { 11 | 'role': 'user', 12 | 'content': f''' 13 | You must complete the python function I give you. You will write the completion in the following form: 14 | 15 | ${{ORIG_FUNCTION}} 16 | <|start_of_completion|> 17 | ${{INSERT_COMPLETION}} 18 | 19 | ORIG_FUNCTION= 20 | {func} 21 | 22 | Please follow the template by repeating the original function, including the <|start_of_completion|> token, then writing the completion. 23 | ''' 24 | } 25 | ] 26 | 27 | def gpt_3_5_prompt(func: str) -> List[dict]: 28 | return [ 29 | { 30 | 'role': 'system', 31 | 'content': f''' 32 | You are an intelligent programmer. You must complete the python function given to you by the user. And you must follow the format they present when giving your answer! 33 | 34 | You can only respond with comments and actual code and the <|start_of_completion|> token, no free-flowing text (unless in a comment).''' 35 | }, 36 | { 37 | 'role': 'user', 38 | 'content': f''' 39 | You must complete the python function I give you. When doing so, you must write the completion int he following form: 40 | ${{ORIG_FUNCTION}} 41 | <|start_of_completion|> 42 | ${{INSERT_COMPLETION}} 43 | 44 | Be sure to use the same indentation I specified. Furthermore, you may only write your response in code/comments. 45 | 46 | ORIG_FUNCTION= 47 | {func} 48 | 49 | Once more, please follow the template by repeating the original function, including the <|start_of_completion|> token, then writing the completion. 50 | ''' 51 | } 52 | ] 53 | 54 | 55 | def claude_prompt(func: str) -> str: 56 | return f''' 57 | Human: You are an intelligent programmer. You must complete the python function I give you. 58 | 59 | Here is the function you must complete: 60 | ```python 61 | {func} 62 | ``` 63 | 64 | Now you must complete it. 65 | 66 | Assistant: I shall complete the python function you gave me to be completely accurate: 67 | ```python 68 | {func.strip()}''' 69 | 70 | def azure_prompt(func: str) -> str: 71 | return f''' 72 | <|im_start|>system<|im_sep|> 73 | You are an intelligent programmer. You must complete the python function the user gives you 74 | <|im_end|> 75 | <|im_start|>user<|im_sep|> 76 | Here is the python function you must complete: 77 | ``` 78 | {func} 79 | ``` 80 | 81 | Now you must complete it 82 | <|im_end|> 83 | <|im_start|>assistant<|im_sep|> 84 | I shall complete the python function you gave me to be completely accurate: 85 | ```python 86 | {func.strip()}''' 87 | 88 | def completion_prompt(func: str) -> str: 89 | return f''' 90 | The following is a conversation between a human and a very intelligent programming assistant. 91 | 92 | Human: 93 | You must complete the python function I specify below 94 | Here it is: 95 | ```python 96 | {func} 97 | ``` 98 | 99 | Assistant: 100 | I shall complete the python function you gave me to be completely accurate: 101 | ```python 102 | {func.strip()}''' -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | fire 3 | numpy 4 | openai 5 | aiohttp 6 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | """ 2 | Runs the human eval dataset through code-davinci for a pass@k evaluation. 3 | 4 | I didn't test pass@100 because I started running into the rate limit for 5 | tokens/minute. 6 | 7 | """ 8 | 9 | import asyncio 10 | import json 11 | import os 12 | import re 13 | import tqdm 14 | import time 15 | import openai 16 | 17 | from dotenv import load_dotenv 18 | from azure import azure_complete 19 | from prompts import ( 20 | gpt4_prompt, 21 | gpt_3_5_prompt, 22 | azure_prompt, 23 | claude_prompt 24 | ) 25 | from claude import claude_complete 26 | 27 | load_dotenv() 28 | 29 | HEADERS = { 30 | "Authorization": f"Bearer {os.environ['OPENAI_API_KEY']}", 31 | "Content-Type": "application/json", 32 | } 33 | 34 | HUMAN_EVAL = os.environ['PWD'] + '/data/HumanEval.jsonl' 35 | OUT_FILE = os.environ['PWD'] + '/results/results-{}-{}.jsonl' 36 | 37 | async def retry(sem, fn): 38 | for i in range(1, 3): 39 | try: 40 | async with sem: 41 | return await fn() 42 | except Exception as e: 43 | print(e) 44 | print('retrying') 45 | time.sleep(0.3*i) 46 | 47 | return await fn() 48 | 49 | async def get_completion(sem, prompt, num_tries=1, model='code-davinci-002', num_errors=0): 50 | #print(num_tries) 51 | if num_tries == 1: 52 | temperature = 0.0 53 | elif num_tries == 10: 54 | temperature = 0.6 55 | elif num_tries == 100: 56 | temperature = 0.8 57 | else: 58 | raise ValueError("num_tries must be 1, 10, or 100") 59 | 60 | 61 | if model in {'gpt-3.5-turbo', 'gpt-4'}: 62 | completion = await retry(sem, lambda: openai.ChatCompletion.acreate(messages=prompt, model=model, temperature=temperature, max_tokens=1000, n=num_tries)) 63 | choices = completion.choices 64 | return [choice['message']['content'] for choice in choices] 65 | elif model in {'azure-gpt-3.5-turbo'}: 66 | completion = await retry(sem, lambda: azure_complete(prompt)) 67 | return [choice['text'] for choice in completion['choices']] 68 | elif 'claude' in model: 69 | return await retry(sem, lambda: claude_complete(prompt, model)) 70 | else: 71 | completion = await retry(sem, lambda: openai.Completion.acreate(prompt=prompt, model=model, temperature=temperature, max_tokens=1000, n=num_tries)) 72 | choices = completion.choices 73 | return [choice['text'] for choice in choices] 74 | 75 | 76 | def iter_hval(): 77 | all_lines = [] 78 | with open(HUMAN_EVAL) as f: 79 | for line in f: 80 | all_lines.append(json.loads(line)) 81 | 82 | return all_lines 83 | 84 | async def get_results(num_tries=10, model='gpt-4'): 85 | out_file = OUT_FILE.format(model, num_tries) 86 | 87 | with open(out_file, 'w') as f: 88 | pass 89 | 90 | sem = asyncio.Semaphore(10) 91 | 92 | async def output(prompt, task_id): 93 | 94 | if model == 'gpt-3.5-turbo': 95 | full_prompt = gpt_3_5_prompt(prompt) 96 | elif model == 'gpt-4': 97 | full_prompt = gpt4_prompt(prompt) 98 | elif model == 'azure-gpt-3.5-turbo': 99 | full_prompt = azure_prompt(prompt) 100 | elif 'claude' in model: 101 | full_prompt = claude_prompt(prompt) 102 | 103 | async with sem: 104 | completions = await get_completion(sem, full_prompt, num_tries=num_tries, model=model) 105 | 106 | outs = [] 107 | # print(completions) 108 | for idx, completion in enumerate(completions): 109 | if '<|start_of_completion|>' in completion: 110 | completion = completion.split('<|start_of_completion|>')[1] 111 | else: 112 | pass 113 | # print('no <|start_of_completion|> token') 114 | # print(completion) 115 | # print('______') 116 | # print(prompt) 117 | # print('') 118 | outs.append({'task_id': task_id, 'completion': completion}) 119 | 120 | return outs 121 | 122 | 123 | futures = [] 124 | for line in tqdm.tqdm(iter_hval()): 125 | prompt = line['prompt'] 126 | task_id = line['task_id'] 127 | futures.append(output(prompt, task_id)) 128 | 129 | with open(out_file, 'a') as out_f: 130 | for future in tqdm.tqdm(asyncio.as_completed(futures), total=len(futures)): 131 | outs = await future 132 | for out in outs: 133 | out_f.write(json.dumps(out) + '\n') 134 | 135 | remove_bloat(out_file) 136 | 137 | 138 | def remove_bloat(in_jsonl): 139 | new_results = [] 140 | with open(in_jsonl, 'r') as f: 141 | for line in f: 142 | out = json.loads(line) 143 | special_token = re.search('\<\|\S+\|\>', out['completion']) 144 | 145 | if special_token: 146 | print(special_token) 147 | out['completion'] = out['completion'][:special_token.start()] 148 | 149 | stop_token = re.search('\n\S', out['completion']) 150 | if stop_token: 151 | out['completion'] = out['completion'][:stop_token.start()] 152 | 153 | 154 | new_results.append(out) 155 | 156 | with open(in_jsonl, 'w') as f: 157 | for result in new_results: 158 | f.write(json.dumps(result) + '\n') 159 | 160 | if __name__ == '__main__': 161 | num_tries=1 162 | # model = 'gpt-4' 163 | model = 'azure-gpt-3.5-turbo' 164 | # model = 'claude-v1.3' 165 | # model = 'claude-instant-v1.1' 166 | # model = 'gpt-3.5-turbo' 167 | 168 | asyncio.run(get_results(num_tries=num_tries, model=model)) 169 | 170 | out_f = OUT_FILE.format(model, num_tries) 171 | remove_bloat(out_f) 172 | print(out_f) 173 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pkg_resources 4 | from setuptools import setup, find_packages 5 | 6 | 7 | setup( 8 | name="human-eval", 9 | py_modules=["human-eval"], 10 | version="1.0", 11 | description="", 12 | author="OpenAI", 13 | packages=find_packages(), 14 | install_requires=[ 15 | str(r) 16 | for r in pkg_resources.parse_requirements( 17 | open(os.path.join(os.path.dirname(__file__), "requirements.txt")) 18 | ) 19 | ], 20 | entry_points={ 21 | "console_scripts": [ 22 | "evaluate_functional_correctness = human_eval.evaluate_functional_correctness", 23 | ] 24 | } 25 | ) 26 | --------------------------------------------------------------------------------