diff --git a/apps/challenges/challenge_config_utils.py b/apps/challenges/challenge_config_utils.py index 5e0c2a3714..10f064936c 100644 --- a/apps/challenges/challenge_config_utils.py +++ b/apps/challenges/challenge_config_utils.py @@ -312,6 +312,7 @@ def get_value_from_field(data, base_location, field_name): "challenge_metadata_schema_errors": "ERROR: Unable to serialize the challenge because of the following errors: {}.", "evaluation_script_not_zip": "ERROR: Please pass in a zip file as evaluation script. If using the `evaluation_script` directory (recommended), it should be `evaluation_script.zip`.", "docker_based_challenge": "ERROR: New Docker based challenges are not supported starting March 15, 2025.", + "invalid_github_branch_format": "ERROR: GitHub branch name '{branch}' is invalid. It must match the pattern 'challenge--' (e.g., challenge-2024-1, challenge-2060-v2).", } @@ -364,6 +365,31 @@ def __init__( self.phase_ids = [] self.leaderboard_ids = [] + def validate_github_branch_format(self): + """ + Ensure the github branch name matches challenge-- + For new challenges, enforce strict format. For existing challenges, allow "challenge" fallback. + """ + branch = self.request.data.get("GITHUB_BRANCH_NAME", "challenge") + pattern = r"^challenge-\d{4}-[a-zA-Z0-9]+$" + + # For new challenge creation (when current_challenge is None), enforce strict format + if not self.current_challenge: + if not re.match(pattern, branch): + self.error_messages.append( + self.error_messages_dict[ + "invalid_github_branch_format" + ].format(branch=branch) + ) + else: + # For existing challenges, allow "challenge" fallback but still validate other formats + if branch != "challenge" and not re.match(pattern, branch): + self.error_messages.append( + self.error_messages_dict[ + "invalid_github_branch_format" + ].format(branch=branch) + ) + def read_and_validate_yaml(self): if not self.yaml_file_count: message = self.error_messages_dict.get("no_yaml_file") @@ -586,6 +612,9 @@ def validate_serializer(self): "github_repository": self.request.data[ "GITHUB_REPOSITORY" ], + "github_branch": self.request.data.get( + "GITHUB_BRANCH_NAME", "challenge" + ), }, ) if not serializer.is_valid(): @@ -1131,6 +1160,9 @@ def validate_challenge_config_util( val_config_util.validate_serializer() + # Add branch format validation + val_config_util.validate_github_branch_format() + # Get existing config IDs for leaderboards and dataset splits if current_challenge: current_challenge_phases = ChallengePhase.objects.filter( diff --git a/apps/challenges/migrations/0113_add_github_branch_field_and_unique_constraint.py b/apps/challenges/migrations/0113_add_github_branch_field_and_unique_constraint.py index 4ff6d35e3b..6a315f6379 100644 --- a/apps/challenges/migrations/0113_add_github_branch_field_and_unique_constraint.py +++ b/apps/challenges/migrations/0113_add_github_branch_field_and_unique_constraint.py @@ -3,20 +3,6 @@ from django.db import migrations, models -def fix_duplicate_github_fields(apps, schema_editor): - """ - No data migration needed since we're using a partial unique constraint. - """ - pass - - -def reverse_fix_duplicate_github_fields(apps, schema_editor): - """ - No reverse migration needed. - """ - pass - - class Migration(migrations.Migration): dependencies = [ @@ -28,13 +14,9 @@ class Migration(migrations.Migration): model_name="challenge", name="github_branch", field=models.CharField( - blank=True, default="", max_length=200, null=True + blank=True, default="challenge", max_length=200, null=True ), ), - migrations.RunPython( - fix_duplicate_github_fields, - reverse_fix_duplicate_github_fields, - ), # Add a partial unique constraint that only applies when both fields are not empty migrations.RunSQL( "CREATE UNIQUE INDEX challenge_github_repo_branch_partial_idx ON challenge (github_repository, github_branch) WHERE github_repository != '' AND github_branch != '';", diff --git a/apps/challenges/models.py b/apps/challenges/models.py index 823309d9d1..acdcd5925a 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -188,7 +188,7 @@ def __init__(self, *args, **kwargs): ) # The github branch name used to create/update the challenge github_branch = models.CharField( - max_length=200, null=True, blank=True, default="" + max_length=200, null=True, blank=True, default="challenge" ) # The number of vCPU for a Fargate worker for the challenge. Default value # is 0.25 vCPU. diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 4e5ac241f3..8c047669f8 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -3897,6 +3897,9 @@ def create_or_update_github_challenge(request, challenge_host_team_pk): response_data = {"error": "ChallengeHostTeam does not exist"} return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) + # Get branch name - required for GitHub-based challenges + # Uses GITHUB_BRANCH_NAME with fallback to "challenge" for backward compatibility + github_branch = request.data.get("GITHUB_BRANCH_NAME", "challenge") challenge_queryset = Challenge.objects.filter( github_repository=request.data["GITHUB_REPOSITORY"], ) @@ -4283,6 +4286,8 @@ def create_or_update_github_challenge(request, challenge_host_team_pk): "challenge_evaluation_script_file" ], "worker_image_url": worker_image_url, + "github_repository": request.data["GITHUB_REPOSITORY"], + "github_branch": github_branch, }, ) if serializer.is_valid(): diff --git a/scripts/migration/populate_github_branch.py b/scripts/migration/populate_github_branch.py new file mode 100644 index 0000000000..99aaaa5858 --- /dev/null +++ b/scripts/migration/populate_github_branch.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# Command to run: python manage.py shell < scripts/migration/populate_github_branch.py +""" +Populate existing challenges with github_branch="challenge" for backward compatibility. + +This script should be run after the migration to ensure all existing challenges +have the github_branch field populated with the default value. +""" + +import traceback + +from challenges.models import Challenge +from django.db import models + + +def populate_github_branch_fields(): + """ + Populate existing challenges with empty github_branch fields to use "challenge" as default. + """ + print("Starting github_branch field population...") + + challenges_to_update = ( + Challenge.objects.filter(github_repository__isnull=False) + .exclude(github_repository="") + .filter( + models.Q(github_branch__isnull=True) | models.Q(github_branch="") + ) + ) + + count = challenges_to_update.count() + + if count == 0: + print("No challenges found that need github_branch population.") + return + + print(f"Found {count} challenges that need github_branch population.") + + updated_count = challenges_to_update.update(github_branch="challenge") + + print( + f"Successfully updated {updated_count} challenges with github_branch='challenge'" + ) + + remaining_empty = ( + Challenge.objects.filter(github_repository__isnull=False) + .exclude(github_repository="") + .filter( + models.Q(github_branch__isnull=True) | models.Q(github_branch="") + ) + .count() + ) + + if remaining_empty == 0: + print("✅ All challenges now have github_branch populated!") + else: + print( + f"⚠️ Warning: {remaining_empty} challenges still have empty github_branch fields" + ) + + sample_challenges = ( + Challenge.objects.filter(github_repository__isnull=False) + .exclude(github_repository="") + .values("id", "title", "github_repository", "github_branch")[:5] + ) + + print("\nSample updated challenges:") + for challenge in sample_challenges: + print( + f" ID: {challenge['id']}, Title: {challenge['title']}, " + f"Repo: {challenge['github_repository']}, Branch: {challenge['github_branch']}" + ) + + +try: + populate_github_branch_fields() + print("\n✅ Script completed successfully!") +except Exception as e: + print(f"\n❌ Error occurred: {e}") + print(traceback.print_exc()) diff --git a/scripts/seed.py b/scripts/seed.py index 0c2cba7f52..4cfe1eddda 100644 --- a/scripts/seed.py +++ b/scripts/seed.py @@ -256,6 +256,7 @@ def create_challenge( featured=is_featured, image=image_file, github_repository=f"evalai-examples/{slug}", + github_branch="challenge", ) challenge.save() diff --git a/tests/unit/challenges/test_views.py b/tests/unit/challenges/test_views.py index 7aaaeada20..68b2ba5a83 100644 --- a/tests/unit/challenges/test_views.py +++ b/tests/unit/challenges/test_views.py @@ -2317,7 +2317,7 @@ def setUp(self): permissions=ChallengeHost.ADMIN, ) - self.challenge = Challenge.objects.create( + self.challenge1 = Challenge.objects.create( title="Test Challenge", short_description="Short description for test challenge", description="Description for test challenge", @@ -2334,7 +2334,7 @@ def setUp(self): start_date=timezone.now() - timedelta(days=2), end_date=timezone.now() + timedelta(days=1), approved_by_admin=True, - github_repository="challenge/github_repo", + github_repository="challenge1/github_repo", ) self.challenge2 = Challenge.objects.create( @@ -2614,73 +2614,73 @@ def test_get_challenge_when_mode_is_host(self): expected = [ { - "id": self.challenge.pk, - "title": self.challenge.title, - "short_description": self.challenge.short_description, - "description": self.challenge.description, - "terms_and_conditions": self.challenge.terms_and_conditions, - "submission_guidelines": self.challenge.submission_guidelines, - "evaluation_details": self.challenge.evaluation_details, + "id": self.challenge1.pk, + "title": self.challenge1.title, + "short_description": self.challenge1.short_description, + "description": self.challenge1.description, + "terms_and_conditions": self.challenge1.terms_and_conditions, + "submission_guidelines": self.challenge1.submission_guidelines, + "evaluation_details": self.challenge1.evaluation_details, "image": None, "start_date": "{0}{1}".format( - self.challenge.start_date.isoformat(), "Z" + self.challenge1.start_date.isoformat(), "Z" ).replace("+00:00", ""), "end_date": "{0}{1}".format( - self.challenge.end_date.isoformat(), "Z" + self.challenge1.end_date.isoformat(), "Z" ).replace("+00:00", ""), "creator": { - "id": self.challenge.creator.pk, - "team_name": self.challenge.creator.team_name, - "created_by": self.challenge.creator.created_by.username, - "team_url": self.challenge.creator.team_url, + "id": self.challenge1.creator.pk, + "team_name": self.challenge1.creator.team_name, + "created_by": self.challenge1.creator.created_by.username, + "team_url": self.challenge1.creator.team_url, }, - "domain": self.challenge.domain, + "domain": self.challenge1.domain, "domain_name": "Computer Vision", - "list_tags": self.challenge.list_tags, - "has_prize": self.challenge.has_prize, - "has_sponsors": self.challenge.has_sponsors, - "published": self.challenge.published, - "submission_time_limit": self.challenge.submission_time_limit, - "is_registration_open": self.challenge.is_registration_open, - "enable_forum": self.challenge.enable_forum, - "leaderboard_description": self.challenge.leaderboard_description, - "anonymous_leaderboard": self.challenge.anonymous_leaderboard, - "manual_participant_approval": self.challenge.manual_participant_approval, + "list_tags": self.challenge1.list_tags, + "has_prize": self.challenge1.has_prize, + "has_sponsors": self.challenge1.has_sponsors, + "published": self.challenge1.published, + "submission_time_limit": self.challenge1.submission_time_limit, + "is_registration_open": self.challenge1.is_registration_open, + "enable_forum": self.challenge1.enable_forum, + "leaderboard_description": self.challenge1.leaderboard_description, + "anonymous_leaderboard": self.challenge1.anonymous_leaderboard, + "manual_participant_approval": self.challenge1.manual_participant_approval, "is_active": True, "allowed_email_domains": [], "blocked_email_domains": [], "banned_email_ids": [], "approved_by_admin": True, - "forum_url": self.challenge.forum_url, - "is_docker_based": self.challenge.is_docker_based, - "is_static_dataset_code_upload": self.challenge.is_static_dataset_code_upload, - "slug": self.challenge.slug, - "max_docker_image_size": self.challenge.max_docker_image_size, - "cli_version": self.challenge.cli_version, - "remote_evaluation": self.challenge.remote_evaluation, - "allow_resuming_submissions": self.challenge.allow_resuming_submissions, - "allow_host_cancel_submissions": self.challenge.allow_host_cancel_submissions, - "allow_cancel_running_submissions": self.challenge.allow_cancel_running_submissions, - "allow_participants_resubmissions": self.challenge.allow_participants_resubmissions, - "workers": self.challenge.workers, + "forum_url": self.challenge1.forum_url, + "is_docker_based": self.challenge1.is_docker_based, + "is_static_dataset_code_upload": self.challenge1.is_static_dataset_code_upload, + "slug": self.challenge1.slug, + "max_docker_image_size": self.challenge1.max_docker_image_size, + "cli_version": self.challenge1.cli_version, + "remote_evaluation": self.challenge1.remote_evaluation, + "allow_resuming_submissions": self.challenge1.allow_resuming_submissions, + "allow_host_cancel_submissions": self.challenge1.allow_host_cancel_submissions, + "allow_cancel_running_submissions": self.challenge1.allow_cancel_running_submissions, + "allow_participants_resubmissions": self.challenge1.allow_participants_resubmissions, + "workers": self.challenge1.workers, "created_at": "{0}{1}".format( - self.challenge.created_at.isoformat(), "Z" + self.challenge1.created_at.isoformat(), "Z" ).replace("+00:00", ""), - "queue": self.challenge.queue, + "queue": self.challenge1.queue, "worker_cpu_cores": 512, "worker_memory": 1024, - "cpu_only_jobs": self.challenge.cpu_only_jobs, - "job_cpu_cores": self.challenge.job_cpu_cores, - "job_memory": self.challenge.job_memory, - "uses_ec2_worker": self.challenge.uses_ec2_worker, - "evaluation_module_error": self.challenge.evaluation_module_error, - "ec2_storage": self.challenge.ec2_storage, - "ephemeral_storage": self.challenge.ephemeral_storage, - "worker_image_url": self.challenge.worker_image_url, - "worker_instance_type": self.challenge.worker_instance_type, - "sqs_retention_period": self.challenge.sqs_retention_period, - "github_repository": self.challenge.github_repository, - "github_branch": self.challenge.github_branch, + "cpu_only_jobs": self.challenge1.cpu_only_jobs, + "job_cpu_cores": self.challenge1.job_cpu_cores, + "job_memory": self.challenge1.job_memory, + "uses_ec2_worker": self.challenge1.uses_ec2_worker, + "evaluation_module_error": self.challenge1.evaluation_module_error, + "ec2_storage": self.challenge1.ec2_storage, + "ephemeral_storage": self.challenge1.ephemeral_storage, + "worker_image_url": self.challenge1.worker_image_url, + "worker_instance_type": self.challenge1.worker_instance_type, + "sqs_retention_period": self.challenge1.sqs_retention_period, + "github_repository": self.challenge1.github_repository, + "github_branch": self.challenge1.github_branch, }, { "id": self.challenge2.pk, @@ -5917,6 +5917,45 @@ def setUp(self): self.client.force_authenticate(user=self.user) + def test_create_challenge_using_github_success(self): + self.url = reverse_lazy( + "challenges:create_or_update_github_challenge", + kwargs={"challenge_host_team_pk": self.challenge_host_team.pk}, + ) + + with mock.patch("challenges.views.requests.get") as m: + resp = mock.Mock() + resp.content = self.test_zip_file.read() + resp.status_code = 200 + m.return_value = resp + response = self.client.post( + self.url, + { + "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", + "GITHUB_BRANCH_NAME": "challenge-2025-v1", + "zip_configuration": self.input_zip_file, + }, + format="multipart", + ) + expected = { + "Success": "Challenge Challenge Title has been created successfully and sent for review to EvalAI Admin." + } + + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json(), expected) + self.assertEqual(Challenge.objects.count(), 1) + self.assertEqual(DatasetSplit.objects.count(), 1) + self.assertEqual(Leaderboard.objects.count(), 1) + self.assertEqual(ChallengePhaseSplit.objects.count(), 1) + + # Verify github_branch is properly stored + challenge = Challenge.objects.first() + self.assertEqual( + challenge.github_repository, + "https://github.com/yourusername/repository", + ) + self.assertEqual(challenge.github_branch, "challenge-2025-v1") + def test_create_challenge_using_github_when_challenge_host_team_does_not_exist( self, ): @@ -5955,6 +5994,230 @@ def test_create_challenge_using_github_when_user_is_not_authenticated( self.assertEqual(list(response.data.values())[0], expected["error"]) self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + def test_create_challenge_using_github_without_branch_name(self): + """Test that missing GITHUB_BRANCH_NAME fails for new challenge creation""" + self.url = reverse_lazy( + "challenges:create_or_update_github_challenge", + kwargs={"challenge_host_team_pk": self.challenge_host_team.pk}, + ) + + with mock.patch("challenges.views.requests.get") as m: + resp = mock.Mock() + resp.content = self.test_zip_file.read() + resp.status_code = 200 + m.return_value = resp + response = self.client.post( + self.url, + { + "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", + "zip_configuration": self.input_zip_file, + }, + format="multipart", + ) + + # Should fail because "challenge" doesn't match the required format for new challenges + self.assertEqual(response.status_code, 400) + self.assertIn("error", response.json()) + # The error comes from challenge_config_utils validation + self.assertIn("invalid", str(response.json()["error"])) + + def test_create_challenge_using_github_with_branch_name_challenge(self): + """Test when branch name is 'challenge' - should fail validation for new challenges""" + self.url = reverse_lazy( + "challenges:create_or_update_github_challenge", + kwargs={"challenge_host_team_pk": self.challenge_host_team.pk}, + ) + + with mock.patch("challenges.views.requests.get") as m: + resp = mock.Mock() + resp.content = self.test_zip_file.read() + resp.status_code = 200 + m.return_value = resp + response = self.client.post( + self.url, + { + "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", + "GITHUB_BRANCH_NAME": "challenge", + "zip_configuration": self.input_zip_file, + }, + format="multipart", + ) + + # Should fail because "challenge" doesn't match the required format for new challenges + self.assertEqual(response.status_code, 400) + self.assertIn("error", response.json()) + # The error comes from challenge_config_utils validation + self.assertIn("invalid", str(response.json()["error"])) + + def test_create_challenge_using_github_with_invalid_branch_name(self): + """Test when branch name is an invalid format (e.g., 'xyzabc')""" + self.url = reverse_lazy( + "challenges:create_or_update_github_challenge", + kwargs={"challenge_host_team_pk": self.challenge_host_team.pk}, + ) + + with mock.patch("challenges.views.requests.get") as m: + resp = mock.Mock() + resp.content = self.test_zip_file.read() + resp.status_code = 200 + m.return_value = resp + response = self.client.post( + self.url, + { + "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", + "GITHUB_BRANCH_NAME": "xyzabc", + "zip_configuration": self.input_zip_file, + }, + format="multipart", + ) + + # Should fail validation due to invalid branch format + self.assertEqual(response.status_code, 400) + self.assertIn("error", response.json()) + self.assertIn("invalid", response.json()["error"]) + + def test_create_challenge_using_github_with_valid_branch_format(self): + """Test when branch name follows the correct format (e.g., 'challenge-2025-v2')""" + self.url = reverse_lazy( + "challenges:create_or_update_github_challenge", + kwargs={"challenge_host_team_pk": self.challenge_host_team.pk}, + ) + + with mock.patch("challenges.views.requests.get") as m: + resp = mock.Mock() + resp.content = self.test_zip_file.read() + resp.status_code = 200 + m.return_value = resp + response = self.client.post( + self.url, + { + "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", + "GITHUB_BRANCH_NAME": "challenge-2025-v2", + "zip_configuration": self.input_zip_file, + }, + format="multipart", + ) + expected = { + "Success": "Challenge Challenge Title has been created successfully and sent for review to EvalAI Admin." + } + + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json(), expected) + + # Verify github_branch is properly stored with the correct format + challenge = Challenge.objects.first() + self.assertEqual( + challenge.github_repository, + "https://github.com/yourusername/repository", + ) + self.assertEqual(challenge.github_branch, "challenge-2025-v2") + + def test_create_challenge_using_github_with_other_valid_branch_formats( + self, + ): + """Test various valid branch name formats""" + self.url = reverse_lazy( + "challenges:create_or_update_github_challenge", + kwargs={"challenge_host_team_pk": self.challenge_host_team.pk}, + ) + + valid_branches = [ + "challenge-2024-1", + "challenge-2025-v1", + "challenge-2060-v2", + "challenge-2024-final", + ] + + for branch_name in valid_branches: + with mock.patch("challenges.views.requests.get") as m: + self.test_zip_file.seek(0) + resp = mock.Mock() + resp.content = self.test_zip_file.read() + resp.status_code = 200 + m.return_value = resp + self.test_zip_file.seek(0) + response = self.client.post( + self.url, + { + "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", + "GITHUB_BRANCH_NAME": branch_name, + "zip_configuration": self.test_zip_file, + }, + format="multipart", + ) + expected = { + "Success": "Challenge Challenge Title has been created successfully and sent for review to EvalAI Admin." + } + + self.assertEqual( + response.status_code, + 201, + f"Failed for branch: {branch_name}", + ) + self.assertEqual(response.json(), expected) + + # Verify github_branch is properly stored + challenge = Challenge.objects.first() + self.assertEqual( + challenge.github_repository, + "https://github.com/yourusername/repository", + ) + self.assertEqual(challenge.github_branch, branch_name) + + # Clean up for next iteration + Challenge.objects.all().delete() + + def test_create_challenge_using_github_with_invalid_branch_formats(self): + """Test various invalid branch name formats""" + self.url = reverse_lazy( + "challenges:create_or_update_github_challenge", + kwargs={"challenge_host_team_pk": self.challenge_host_team.pk}, + ) + + invalid_branches = [ + "main", + "master", + "develop", + "feature-branch", + "challenge", + "challenge-2025", + "challenge-v2", + "2025-challenge-v2", + "challenge-2025-v2-extra", + ] + + for branch_name in invalid_branches: + with mock.patch("challenges.views.requests.get") as m: + self.test_zip_file.seek( + 0 + ) # Reset file pointer to the beginning + resp = mock.Mock() + resp.content = self.test_zip_file.read() + resp.status_code = 200 + m.return_value = resp + self.test_zip_file.seek( + 0 + ) # Reset file pointer to the beginning + response = self.client.post( + self.url, + { + "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", + "GITHUB_BRANCH_NAME": branch_name, + "zip_configuration": self.test_zip_file, + }, + format="multipart", + ) + + # Should fail validation due to invalid branch format + self.assertEqual( + response.status_code, + 400, + f"Should fail for branch: {branch_name}", + ) + self.assertIn("error", response.json()) + # The error comes from challenge_config_utils validation + self.assertIn("invalid", str(response.json()["error"])) + class ValidateChallengeTest(APITestCase): def setUp(self): @@ -6021,7 +6284,7 @@ def test_validate_challenge_using_success(self): self.url, { "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", - "GITHUB_BRANCH_NAME": "refs/heads/challenge", + "GITHUB_BRANCH_NAME": "challenge-2025-v1", "zip_configuration": self.input_zip_file, }, format="multipart", @@ -6044,13 +6307,14 @@ def test_validate_challenge_using_failure(self): resp = mock.Mock() resp.content = self.test_zip_incorrect_file.read() resp.status_code = 200 - m.return_value = resp + self.test_zip_incorrect_file.seek(0) response = self.client.post( self.url, { "GITHUB_REPOSITORY": "https://github.com/yourusername/repository", - "zip_configuration": self.input_zip_file, + "GITHUB_BRANCH_NAME": "challenge-2025-v1", + "zip_configuration": self.test_zip_incorrect_file, }, format="multipart", )