featurebench
v1.0FeatureBench full split: 200 feature-implementation tasks across 24 Python repos. 7 tasks require Ampere+ GPU. Original benchmark: https://github.com/LiberCoders/FeatureBench. Adapter: https://github.com/harbor-framework/harbor/pull/875.
uvx harbor run -d featurebench@1.0Tasks (200)
astropy__astropy.b0db0daa.test_basic_rgb.067e927c.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_basic_rgb.067e927c.lv1HEAD
astropy__astropy.b0db0daa.test_basic_rgb.067e927c.lv2
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_basic_rgb.067e927c.lv2HEAD
astropy__astropy.b0db0daa.test_comparison.445d81a3.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_comparison.445d81a3.lv1HEAD
astropy__astropy.b0db0daa.test_compressed.8daeb7d6.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_compressed.8daeb7d6.lv1HEAD
astropy__astropy.b0db0daa.test_compression_failures.48dc420b.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_compression_failures.48dc420b.lv1HEAD
astropy__astropy.b0db0daa.test_containers.6079987d.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_containers.6079987d.lv1HEAD
astropy__astropy.b0db0daa.test_converter.2ef0539f.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_converter.2ef0539f.lv1HEAD
astropy__astropy.b0db0daa.test_functions.40530bf7.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_functions.40530bf7.lv1HEAD
astropy__astropy.b0db0daa.test_lombscargle_multiband.78687278.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_lombscargle_multiband.78687278.lv1HEAD
astropy__astropy.b0db0daa.test_quantity_erfa_ufuncs.3fc54395.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_quantity_erfa_ufuncs.3fc54395.lv1HEAD
astropy__astropy.b0db0daa.test_realizations.5b9cf523.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_realizations.5b9cf523.lv1HEAD
astropy__astropy.b0db0daa.test_table.48eef659.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_table.48eef659.lv1HEAD
astropy__astropy.b0db0daa.test_tilde_path.383d7c0c.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_tilde_path.383d7c0c.lv1HEAD
astropy__astropy.b0db0daa.test_tiled_compression.a4fe298a.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_tiled_compression.a4fe298a.lv1HEAD
astropy__astropy.b0db0daa.test_tree.261bac3c.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_tree.261bac3c.lv1HEAD
astropy__astropy.b0db0daa.test_tree.261bac3c.lv2
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_tree.261bac3c.lv2HEAD
astropy__astropy.b0db0daa.test_vo.8fd473ce.lv1
uvx harbor run -d featurebench@1.0 -t astropy__astropy.b0db0daa.test_vo.8fd473ce.lv1HEAD
fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1
uvx harbor run -d featurebench@1.0 -t fastapi__fastapi.02e108d1.test_compat.71e8518f.lv1HEAD
fastapi__fastapi.02e108d1.test_compat.71e8518f.lv2
uvx harbor run -d featurebench@1.0 -t fastapi__fastapi.02e108d1.test_compat.71e8518f.lv2HEAD
huggingface__accelerate.a73fd3a1.test_big_modeling.a10a845e.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__accelerate.a73fd3a1.test_big_modeling.a10a845e.lv1HEAD
huggingface__transformers.e2e8dbed.test_candidate_generator.bc0ea399.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_candidate_generator.bc0ea399.lv1HEAD
huggingface__transformers.e2e8dbed.test_chat_template_utils.8c88f8ec.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_chat_template_utils.8c88f8ec.lv1HEAD
huggingface__transformers.e2e8dbed.test_feature_extraction_auto.13dd67ea.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_feature_extraction_auto.13dd67ea.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_colpali.370cfbdd.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_colpali.370cfbdd.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_informer.de781186.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_informer.de781186.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_lightglue.3d3d57cb.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_lightglue.3d3d57cb.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_mimi.3118abde.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_mimi.3118abde.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_pixtral.a620bb0b.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_pixtral.a620bb0b.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_pixtral.a620bb0b.lv2
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_pixtral.a620bb0b.lv2HEAD
huggingface__transformers.e2e8dbed.test_modeling_prompt_depth_anything.2d54afdc.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_prompt_depth_anything.2d54afdc.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_seggpt.f9e83f60.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_seggpt.f9e83f60.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_seggpt.f9e83f60.lv2
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_seggpt.f9e83f60.lv2HEAD
huggingface__transformers.e2e8dbed.test_modeling_speech_encoder_decoder.b0d78d76.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_speech_encoder_decoder.b0d78d76.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_speech_to_text.e53c8eb4.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_speech_to_text.e53c8eb4.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_speech_to_text.e53c8eb4.lv2
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_speech_to_text.e53c8eb4.lv2HEAD
huggingface__transformers.e2e8dbed.test_modeling_swin2sr.f5cba486.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_swin2sr.f5cba486.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_swin2sr.f5cba486.lv2
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_swin2sr.f5cba486.lv2HEAD
huggingface__transformers.e2e8dbed.test_modeling_time_series_transformer.b0ef2ff5.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_time_series_transformer.b0ef2ff5.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_vitmatte.2d342386.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_vitmatte.2d342386.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_vitpose.30ffd86e.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_vitpose.30ffd86e.lv1HEAD
huggingface__transformers.e2e8dbed.test_modeling_vitpose.30ffd86e.lv2
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_vitpose.30ffd86e.lv2HEAD
huggingface__transformers.e2e8dbed.test_modeling_vits.6379a2ba.lv2
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_vits.6379a2ba.lv2HEAD
huggingface__transformers.e2e8dbed.test_modeling_zoedepth.e19636d3.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_modeling_zoedepth.e19636d3.lv1HEAD
huggingface__transformers.e2e8dbed.test_processing_git.7342da79.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_processing_git.7342da79.lv1HEAD
huggingface__transformers.e2e8dbed.test_processing_grounding_dino.3d24a7e6.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_processing_grounding_dino.3d24a7e6.lv1HEAD
huggingface__transformers.e2e8dbed.test_processing_owlvit.56ec5dc9.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_processing_owlvit.56ec5dc9.lv1HEAD
huggingface__transformers.e2e8dbed.test_processing_samhq.7732c356.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_processing_samhq.7732c356.lv1HEAD
huggingface__transformers.e2e8dbed.test_processing_wav2vec2.4f660c78.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_processing_wav2vec2.4f660c78.lv1HEAD
huggingface__transformers.e2e8dbed.test_processor_auto.d9626018.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_processor_auto.d9626018.lv1HEAD
huggingface__transformers.e2e8dbed.test_serve.4e7860c7.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_serve.4e7860c7.lv1HEAD
huggingface__transformers.e2e8dbed.test_tests_fetcher.e1abe0dd.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_tests_fetcher.e1abe0dd.lv1HEAD
huggingface__transformers.e2e8dbed.test_tokenization_gpt_sw3.18420a43.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_tokenization_gpt_sw3.18420a43.lv1HEAD
huggingface__transformers.e2e8dbed.test_tokenization_prophetnet.ca02e775.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_tokenization_prophetnet.ca02e775.lv1HEAD
huggingface__transformers.e2e8dbed.test_video_processing_auto.c41f8df4.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__transformers.e2e8dbed.test_video_processing_auto.c41f8df4.lv1HEAD
huggingface__trl.02a34777.test_data_utils.827a9d15.lv1
uvx harbor run -d featurebench@1.0 -t huggingface__trl.02a34777.test_data_utils.827a9d15.lv1HEAD
huggingface__trl.02a34777.test_data_utils.827a9d15.lv2
uvx harbor run -d featurebench@1.0 -t huggingface__trl.02a34777.test_data_utils.827a9d15.lv2HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_connector.825ef3b1.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_connector.825ef3b1.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_data.12056068.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_data.12056068.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_data.c8b292af.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_data.c8b292af.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv2
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_fsdp_integration.61c07610.lv2HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_fsdp.744aa3f5.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_fsdp.744aa3f5.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_hooks.34620588.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_hooks.34620588.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_model_parallel.7dec8f24.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_model_parallel.7dec8f24.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_seed.1eaa1d46.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_seed.1eaa1d46.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_seed.1eaa1d46.lv2
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_seed.1eaa1d46.lv2HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_wrappers.d88ce6f6.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_wrappers.d88ce6f6.lv1HEAD
lightning-ai__pytorch-lightning.126fa6f1.test_xla.7c94e7d2.lv1
uvx harbor run -d featurebench@1.0 -t lightning-ai__pytorch-lightning.126fa6f1.test_xla.7c94e7d2.lv1HEAD
linkedin__liger-kernel.c856fbab.test_fused_add_rms_norm.fecadbdf.lv1
uvx harbor run -d featurebench@1.0 -t linkedin__liger-kernel.c856fbab.test_fused_add_rms_norm.fecadbdf.lv1HEAD
linkedin__liger-kernel.c856fbab.test_fused_linear_jsd.4bd46945.lv1
uvx harbor run -d featurebench@1.0 -t linkedin__liger-kernel.c856fbab.test_fused_linear_jsd.4bd46945.lv1HEAD
linkedin__liger-kernel.c856fbab.test_fused_neighborhood_attention.78217be4.lv2
uvx harbor run -d featurebench@1.0 -t linkedin__liger-kernel.c856fbab.test_fused_neighborhood_attention.78217be4.lv2HEAD
linkedin__liger-kernel.c856fbab.test_layer_norm.ed322799.lv1
uvx harbor run -d featurebench@1.0 -t linkedin__liger-kernel.c856fbab.test_layer_norm.ed322799.lv1HEAD
linkedin__liger-kernel.c856fbab.test_mini_models.96209d55.lv1
uvx harbor run -d featurebench@1.0 -t linkedin__liger-kernel.c856fbab.test_mini_models.96209d55.lv1HEAD
linkedin__liger-kernel.c856fbab.test_multi_token_attention.4c06a35b.lv1
uvx harbor run -d featurebench@1.0 -t linkedin__liger-kernel.c856fbab.test_multi_token_attention.4c06a35b.lv1HEAD
linkedin__liger-kernel.c856fbab.test_poly_norm.7b0e3399.lv1
uvx harbor run -d featurebench@1.0 -t linkedin__liger-kernel.c856fbab.test_poly_norm.7b0e3399.lv1HEAD
matplotlib__matplotlib.86a476d2.test_angle_helper.48e96b5a.lv2
uvx harbor run -d featurebench@1.0 -t matplotlib__matplotlib.86a476d2.test_angle_helper.48e96b5a.lv2HEAD
matplotlib__matplotlib.86a476d2.test_backend_registry.872ba384.lv1
uvx harbor run -d featurebench@1.0 -t matplotlib__matplotlib.86a476d2.test_backend_registry.872ba384.lv1HEAD
mesonbuild__meson.f5d81d07.cargotests.8e49c2d0.lv1
uvx harbor run -d featurebench@1.0 -t mesonbuild__meson.f5d81d07.cargotests.8e49c2d0.lv1HEAD
mesonbuild__meson.f5d81d07.cargotests.8e49c2d0.lv2
uvx harbor run -d featurebench@1.0 -t mesonbuild__meson.f5d81d07.cargotests.8e49c2d0.lv2HEAD
mlflow__mlflow.93dab383.test_abstract_store.e5ff5123.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_abstract_store.e5ff5123.lv1HEAD
mlflow__mlflow.93dab383.test_ai_command_utils.85dcb487.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_ai_command_utils.85dcb487.lv1HEAD
mlflow__mlflow.93dab383.test_ai_command_utils.85dcb487.lv2
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_ai_command_utils.85dcb487.lv2HEAD
mlflow__mlflow.93dab383.test_assessment.df512d1b.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_assessment.df512d1b.lv1HEAD
mlflow__mlflow.93dab383.test_bedrock_autolog.f008b521.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_bedrock_autolog.f008b521.lv1HEAD
mlflow__mlflow.93dab383.test_client_webhooks.75e39b52.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_client_webhooks.75e39b52.lv1HEAD
mlflow__mlflow.93dab383.test_client_webhooks.75e39b52.lv2
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_client_webhooks.75e39b52.lv2HEAD
mlflow__mlflow.93dab383.test_client.42de781c.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_client.42de781c.lv1HEAD
mlflow__mlflow.93dab383.test_config.c63d41b0.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_config.c63d41b0.lv1HEAD
mlflow__mlflow.93dab383.test_config.c63d41b0.lv2
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_config.c63d41b0.lv2HEAD
mlflow__mlflow.93dab383.test_databricks_adapter.3b72f337.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_databricks_adapter.3b72f337.lv1HEAD
mlflow__mlflow.93dab383.test_databricks_rest_store.50525c62.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_databricks_rest_store.50525c62.lv1HEAD
mlflow__mlflow.93dab383.test_databricks_tracing_utils.8ef44eb4.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_databricks_tracing_utils.8ef44eb4.lv1HEAD
mlflow__mlflow.93dab383.test_databricks.cbc54e8c.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_databricks.cbc54e8c.lv1HEAD
mlflow__mlflow.93dab383.test_dependencies_schema.7019486f.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_dependencies_schema.7019486f.lv1HEAD
mlflow__mlflow.93dab383.test_evaluation_dataset.4a6528c2.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_evaluation_dataset.4a6528c2.lv1HEAD
mlflow__mlflow.93dab383.test_file_store_logged_model.a9596c54.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_file_store_logged_model.a9596c54.lv1HEAD
mlflow__mlflow.93dab383.test_file_store_logged_model.a9596c54.lv2
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_file_store_logged_model.a9596c54.lv2HEAD
mlflow__mlflow.93dab383.test_genai_eval_utils.2d5441f2.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_genai_eval_utils.2d5441f2.lv1HEAD
mlflow__mlflow.93dab383.test_genai_metrics.5f107830.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_genai_metrics.5f107830.lv1HEAD
mlflow__mlflow.93dab383.test_git_versioning.b91d0418.lv1
uvx harbor run -d featurebench@1.0 -t mlflow__mlflow.93dab383.test_git_versioning.b91d0418.lv1HEAD