use crate::tools::edit_file_tool::*;
use crate::{
    AgentTool, ContextServerRegistry, EditFileTool, GrepTool, GrepToolInput, ReadFileTool,
    ReadFileToolInput, Template, Templates, Thread, ToolCallEventStream, ToolInput,
};
use Role::*;
use anyhow::{Context as _, Result};
use client::{Client, RefreshLlmTokenListener, UserStore};
use fs::FakeFs;
use futures::{FutureExt, StreamExt, future::LocalBoxFuture};
use gpui::{AppContext as _, AsyncApp, Entity, TestAppContext, UpdateGlobal as _};
use http_client::StatusCode;
use language::language_settings::FormatOnSave;
use language_model::{
    LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
    LanguageModelRegistry, LanguageModelRequest, LanguageModelRequestMessage,
    LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse,
    LanguageModelToolUseId, MessageContent, Role, SelectedModel,
};
use project::Project;
use prompt_store::{ProjectContext, WorktreeContext};
use rand::prelude::*;
use reqwest_client::ReqwestClient;
use serde::Serialize;
use serde_json::json;
use settings::SettingsStore;
use std::{
    fmt::{self, Display},
    path::{Path, PathBuf},
    str::FromStr,
    sync::Arc,
    time::Duration,
};
use util::path;

#[derive(Serialize)]
struct DiffJudgeTemplate {
    diff: String,
    assertions: &'static str,
}

impl Template for DiffJudgeTemplate {
    const TEMPLATE_NAME: &'static str = "diff_judge.hbs";
}

#[derive(Clone)]
struct EvalInput {
    conversation: Vec<LanguageModelRequestMessage>,
    input_file_path: PathBuf,
    input_content: Option<String>,
    assertion: EvalAssertion,
}

impl EvalInput {
    fn new(
        conversation: Vec<LanguageModelRequestMessage>,
        input_file_path: impl Into<PathBuf>,
        input_content: Option<String>,
        assertion: EvalAssertion,
    ) -> Self {
        EvalInput {
            conversation,
            input_file_path: input_file_path.into(),
            input_content,
            assertion,
        }
    }
}

#[derive(Clone)]
struct EvalSample {
    text_before: String,
    text_after: String,
    tool_input: EditFileToolInput,
    diff: String,
}

trait AssertionFn: 'static + Send + Sync {
    fn assert<'a>(
        &'a self,
        sample: &'a EvalSample,
        judge_model: Arc<dyn LanguageModel>,
        cx: &'a mut TestAppContext,
    ) -> LocalBoxFuture<'a, Result<EvalAssertionOutcome>>;
}

impl<F> AssertionFn for F
where
    F: 'static
        + Send
        + Sync
        + AsyncFn(
            &EvalSample,
            Arc<dyn LanguageModel>,
            &mut TestAppContext,
        ) -> Result<EvalAssertionOutcome>,
{
    fn assert<'a>(
        &'a self,
        sample: &'a EvalSample,
        judge_model: Arc<dyn LanguageModel>,
        cx: &'a mut TestAppContext,
    ) -> LocalBoxFuture<'a, Result<EvalAssertionOutcome>> {
        (self)(sample, judge_model, cx).boxed_local()
    }
}

#[derive(Clone)]
struct EvalAssertion(Arc<dyn AssertionFn>);

impl EvalAssertion {
    fn new<F>(f: F) -> Self
    where
        F: 'static
            + Send
            + Sync
            + AsyncFn(
                &EvalSample,
                Arc<dyn LanguageModel>,
                &mut TestAppContext,
            ) -> Result<EvalAssertionOutcome>,
    {
        EvalAssertion(Arc::new(f))
    }

    fn assert_diff_any(expected_diffs: Vec<impl Into<String>>) -> Self {
        let expected_diffs: Vec<String> = expected_diffs.into_iter().map(Into::into).collect();
        Self::new(async move |sample, _judge, _cx| {
            let matches = expected_diffs.iter().any(|possible_diff| {
                language::apply_diff_patch(&sample.text_before, possible_diff)
                    .map(|expected| {
                        strip_empty_lines(&expected) == strip_empty_lines(&sample.text_after)
                    })
                    .unwrap_or(false)
            });

            Ok(EvalAssertionOutcome {
                score: if matches { 100 } else { 0 },
                message: None,
            })
        })
    }

    fn judge_diff(assertions: &'static str) -> Self {
        Self::new(async move |sample, judge, cx| {
            let prompt = DiffJudgeTemplate {
                diff: sample.diff.clone(),
                assertions,
            }
            .render(&Templates::new())
            .context("Failed to render diff judge template")?;

            let request = LanguageModelRequest {
                messages: vec![LanguageModelRequestMessage {
                    role: Role::User,
                    content: vec![prompt.into()],
                    cache: false,
                    reasoning_details: None,
                }],
                thinking_allowed: true,
                thinking_effort: judge
                    .default_effort_level()
                    .map(|effort_level| effort_level.value.to_string()),
                ..Default::default()
            };
            let mut response = retry_on_rate_limit(async || {
                Ok(judge
                    .stream_completion_text(request.clone(), &cx.to_async())
                    .await?)
            })
            .await?;
            let mut output = String::new();
            while let Some(chunk) = response.stream.next().await {
                let chunk = chunk?;
                output.push_str(&chunk);
            }

            let re = regex::Regex::new(r"<score>(\d+)</score>")
                .context("Failed to compile score regex")?;
            if let Some(captures) = re.captures(&output)
                && let Some(score_match) = captures.get(1)
            {
                let score = score_match.as_str().parse().unwrap_or(0);
                return Ok(EvalAssertionOutcome {
                    score,
                    message: Some(output),
                });
            }

            anyhow::bail!("No score found in response. Raw output: {output}");
        })
    }

    async fn run(
        &self,
        input: &EvalSample,
        judge_model: Arc<dyn LanguageModel>,
        cx: &mut TestAppContext,
    ) -> Result<EvalAssertionOutcome> {
        self.0.assert(input, judge_model, cx).await
    }
}

#[derive(Clone)]
struct EditEvalOutput {
    sample: EvalSample,
    assertion: EvalAssertionOutcome,
}

impl Display for EditEvalOutput {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        writeln!(f, "Score: {:?}", self.assertion.score)?;
        if let Some(message) = self.assertion.message.as_ref() {
            writeln!(f, "Message: {}", message)?;
        }
        writeln!(f, "Diff:\n{}", self.sample.diff)?;
        writeln!(f, "Tool Input:\n{:#?}", self.sample.tool_input)?;
        Ok(())
    }
}

#[derive(Clone, Debug, Eq, PartialEq, Hash)]
struct EvalAssertionOutcome {
    score: usize,
    message: Option<String>,
}

struct EditToolTest {
    fs: Arc<FakeFs>,
    project: Entity<Project>,
    model: Arc<dyn LanguageModel>,
    judge_model: Arc<dyn LanguageModel>,
    model_thinking_effort: Option<String>,
}

impl EditToolTest {
    async fn new(cx: &mut TestAppContext) -> Self {
        cx.executor().allow_parking();

        let fs = FakeFs::new(cx.executor());
        cx.update(|cx| {
            let settings_store = SettingsStore::test(cx);
            cx.set_global(settings_store);
            SettingsStore::update_global(cx, |store: &mut SettingsStore, cx| {
                store.update_user_settings(cx, |settings| {
                    settings
                        .project
                        .all_languages
                        .defaults
                        .ensure_final_newline_on_save = Some(false);
                    settings.project.all_languages.defaults.format_on_save =
                        Some(FormatOnSave::Off);
                });
            });

            gpui_tokio::init(cx);
            let http_client = Arc::new(ReqwestClient::user_agent("agent tests").unwrap());
            cx.set_http_client(http_client);
            let client = Client::production(cx);
            let user_store = cx.new(|cx| UserStore::new(client.clone(), cx));
            language_model::init(cx);
            RefreshLlmTokenListener::register(client.clone(), user_store.clone(), cx);
            language_models::init(user_store, client, cx);
        });

        fs.insert_tree("/root", json!({})).await;
        let project = Project::test(fs.clone(), [path!("/root").as_ref()], cx).await;
        let agent_model = SelectedModel::from_str(
            &std::env::var("ZED_AGENT_MODEL")
                .unwrap_or("anthropic/claude-sonnet-4-6-latest".into()),
        )
        .unwrap();
        let judge_model = SelectedModel::from_str(
            &std::env::var("ZED_JUDGE_MODEL")
                .unwrap_or("anthropic/claude-sonnet-4-6-latest".into()),
        )
        .unwrap();

        let authenticate_provider_tasks = cx.update(|cx| {
            LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
                registry
                    .providers()
                    .iter()
                    .map(|p| p.authenticate(cx))
                    .collect::<Vec<_>>()
            })
        });
        let (model, judge_model) = cx
            .update(|cx| {
                cx.spawn(async move |cx| {
                    futures::future::join_all(authenticate_provider_tasks).await;
                    let model = Self::load_model(&agent_model, cx).await;
                    let judge_model = Self::load_model(&judge_model, cx).await;
                    (model.unwrap(), judge_model.unwrap())
                })
            })
            .await;

        let model_thinking_effort = model
            .default_effort_level()
            .map(|effort_level| effort_level.value.to_string());

        Self {
            fs,
            project,
            model,
            judge_model,
            model_thinking_effort,
        }
    }

    async fn load_model(
        selected_model: &SelectedModel,
        cx: &mut AsyncApp,
    ) -> Result<Arc<dyn LanguageModel>> {
        cx.update(|cx| {
            let registry = LanguageModelRegistry::read_global(cx);
            let provider = registry
                .provider(&selected_model.provider)
                .expect("Provider not found");
            provider.authenticate(cx)
        })
        .await?;
        Ok(cx.update(|cx| {
            let models = LanguageModelRegistry::read_global(cx);
            models
                .available_models(cx)
                .find(|model| {
                    model.provider_id() == selected_model.provider
                        && model.id() == selected_model.model
                })
                .unwrap_or_else(|| panic!("Model {} not found", selected_model.model.0))
        }))
    }

    async fn eval(&self, mut eval: EvalInput, cx: &mut TestAppContext) -> Result<EditEvalOutput> {
        eval.conversation
            .last_mut()
            .context("Conversation must not be empty")?
            .cache = true;

        // Populate the FakeFs so `resolve_path` / `entry_for_path` can find
        // the file in the worktree.
        if let Some(input_content) = eval.input_content.as_deref() {
            let abs_path = Path::new("/root").join(
                eval.input_file_path
                    .strip_prefix("root")
                    .unwrap_or(&eval.input_file_path),
            );
            self.fs.insert_file(&abs_path, input_content.into()).await;

            // Wait for the worktree to pick up the new file.
            cx.run_until_parked();
        }

        let tools = crate::built_in_tools().collect::<Vec<_>>();

        let system_prompt = {
            let worktrees = vec![WorktreeContext {
                root_name: "root".to_string(),
                abs_path: Path::new("/path/to/root").into(),
                rules_file: None,
            }];
            let project_context = ProjectContext::new(worktrees);
            let tool_names = tools
                .iter()
                .map(|tool| tool.name.clone().into())
                .collect::<Vec<_>>();
            let template = crate::SystemPromptTemplate {
                project: &project_context,
                available_tools: tool_names,
                model_name: None,
                date: chrono::Local::now().format("%Y-%m-%d").to_string(),
                user_agents_md: None,
            };
            let templates = Templates::new();
            template.render(&templates)?
        };

        let has_system_prompt = eval
            .conversation
            .first()
            .is_some_and(|msg| msg.role == Role::System);
        let messages = if has_system_prompt {
            eval.conversation
        } else {
            [LanguageModelRequestMessage {
                role: Role::System,
                content: vec![MessageContent::Text(system_prompt)],
                cache: true,
                reasoning_details: None,
            }]
            .into_iter()
            .chain(eval.conversation)
            .collect::<Vec<_>>()
        };

        let request = LanguageModelRequest {
            messages,
            tools,
            thinking_allowed: true,
            thinking_effort: self.model_thinking_effort.clone(),
            ..Default::default()
        };

        // The model will call the tool as "edit_file" (the production-visible
        // name), but the schema is from EditFileTool.
        let tool_input =
            retry_on_rate_limit(async || self.extract_tool_use(request.clone(), cx).await).await?;

        let language_registry = self
            .project
            .read_with(cx, |project, _cx| project.languages().clone());

        let context_server_registry = cx
            .new(|cx| ContextServerRegistry::new(self.project.read(cx).context_server_store(), cx));
        let thread = cx.new(|cx| {
            Thread::new(
                self.project.clone(),
                cx.new(|_cx| ProjectContext::default()),
                context_server_registry,
                Templates::new(),
                Some(self.model.clone()),
                cx,
            )
        });
        let action_log = thread.read_with(cx, |thread, _| thread.action_log().clone());

        let tool = Arc::new(EditFileTool::new(
            self.project.clone(),
            thread.downgrade(),
            action_log,
            language_registry,
        ));

        let result = cx
            .update(|cx| {
                tool.clone().run(
                    ToolInput::resolved(tool_input.clone()),
                    ToolCallEventStream::test().0,
                    cx,
                )
            })
            .await;

        let output = match result {
            Ok(output) => output,
            Err(output) => {
                anyhow::bail!("Tool returned error: {}", output);
            }
        };

        let EditFileToolOutput::Success { new_text, .. } = &output else {
            anyhow::bail!("Tool returned error output: {}", output);
        };

        let sample = EvalSample {
            tool_input,
            diff: language::unified_diff(
                eval.input_content.as_deref().unwrap_or_default(),
                new_text,
            ),
            text_before: eval.input_content.unwrap_or_default(),
            text_after: new_text.clone(),
        };

        let assertion = eval
            .assertion
            .run(&sample, self.judge_model.clone(), cx)
            .await?;

        Ok(EditEvalOutput { assertion, sample })
    }

    /// Stream the model completion and extract the first complete tool use
    /// whose name matches `EditFileTool::NAME`, parsed as `EditFileToolInput`.
    async fn extract_tool_use(
        &self,
        request: LanguageModelRequest,
        cx: &mut TestAppContext,
    ) -> Result<EditFileToolInput> {
        let model = self.model.clone();
        let events = cx
            .update(|cx| {
                let async_cx = cx.to_async();
                cx.foreground_executor()
                    .spawn(async move { model.stream_completion(request, &async_cx).await })
            })
            .await
            .map_err(|err| anyhow::anyhow!("completion error: {}", err))?;

        let mut streamed_text = String::new();
        let mut stop_reason = None;
        let mut parse_errors = Vec::new();

        let mut events = events.fuse();
        while let Some(event) = events.next().await {
            match event {
                Ok(LanguageModelCompletionEvent::ToolUse(tool_use))
                    if tool_use.is_input_complete
                        && tool_use.name.as_ref() == EditFileTool::NAME =>
                {
                    let input: EditFileToolInput = serde_json::from_value(tool_use.input)
                        .context("Failed to parse tool input as EditFileToolInput")?;
                    return Ok(input);
                }
                Ok(LanguageModelCompletionEvent::Text(text)) => {
                    if streamed_text.len() < 2_000 {
                        streamed_text.push_str(&text);
                    }
                }
                Ok(LanguageModelCompletionEvent::Stop(reason)) => {
                    stop_reason = Some(reason);
                }
                Ok(LanguageModelCompletionEvent::ToolUseJsonParseError {
                    tool_name,
                    raw_input,
                    json_parse_error,
                    ..
                }) if tool_name.as_ref() == EditFileTool::NAME => {
                    parse_errors.push(format!("{json_parse_error}\nRaw input:\n{raw_input:?}"));
                }
                Err(err) => {
                    return Err(anyhow::anyhow!("completion error: {}", err));
                }
                _ => {}
            }
        }

        let streamed_text = streamed_text.trim();
        let streamed_text_suffix = if streamed_text.is_empty() {
            String::new()
        } else {
            format!("\nStreamed text:\n{streamed_text}")
        };
        let stop_reason_suffix = stop_reason
            .map(|reason| format!("\nStop reason: {reason:?}"))
            .unwrap_or_default();
        let parse_errors_suffix = if parse_errors.is_empty() {
            String::new()
        } else {
            format!("\nTool parse errors:\n{}", parse_errors.join("\n"))
        };

        anyhow::bail!(
            "Stream ended without an edit_file tool use{stop_reason_suffix}{parse_errors_suffix}{streamed_text_suffix}"
        )
    }
}

fn run_eval(eval: EvalInput) -> eval_utils::EvalOutput<()> {
    super::run_gpui_eval(
        |cx| {
            async move {
                let test = EditToolTest::new(cx).await;
                let result = test.eval(eval, cx).await;
                drop(test);
                cx.run_until_parked();
                result
            }
            .boxed_local()
        },
        |output| {
            if output.assertion.score < 80 {
                eval_utils::OutcomeKind::Failed
            } else {
                eval_utils::OutcomeKind::Passed
            }
        },
    )
}

fn message(
    role: Role,
    contents: impl IntoIterator<Item = MessageContent>,
) -> LanguageModelRequestMessage {
    LanguageModelRequestMessage {
        role,
        content: contents.into_iter().collect(),
        cache: false,
        reasoning_details: None,
    }
}

fn text(text: impl Into<String>) -> MessageContent {
    MessageContent::Text(text.into())
}

fn lines(input: &str, range: std::ops::Range<usize>) -> String {
    input
        .lines()
        .skip(range.start)
        .take(range.len())
        .collect::<Vec<_>>()
        .join("\n")
}

fn tool_use(
    id: impl Into<Arc<str>>,
    name: impl Into<Arc<str>>,
    input: impl Serialize,
) -> MessageContent {
    MessageContent::ToolUse(LanguageModelToolUse {
        id: LanguageModelToolUseId::from(id.into()),
        name: name.into(),
        raw_input: serde_json::to_string_pretty(&input).unwrap(),
        input: serde_json::to_value(input).unwrap(),
        is_input_complete: true,
        thought_signature: None,
    })
}

fn tool_result(
    id: impl Into<Arc<str>>,
    name: impl Into<Arc<str>>,
    result: impl Into<Arc<str>>,
) -> MessageContent {
    MessageContent::ToolResult(LanguageModelToolResult {
        tool_use_id: LanguageModelToolUseId::from(id.into()),
        tool_name: name.into(),
        is_error: false,
        content: vec![LanguageModelToolResultContent::Text(result.into())],
        output: None,
    })
}

fn strip_empty_lines(text: &str) -> String {
    text.lines()
        .filter(|line| !line.trim().is_empty())
        .collect::<Vec<_>>()
        .join("\n")
}

async fn retry_on_rate_limit<R>(mut request: impl AsyncFnMut() -> Result<R>) -> Result<R> {
    const MAX_RETRIES: usize = 20;
    let mut attempt = 0;

    loop {
        attempt += 1;
        let response = request().await;

        if attempt >= MAX_RETRIES {
            return response;
        }

        let retry_delay = match &response {
            Ok(_) => None,
            Err(err) => match err.downcast_ref::<LanguageModelCompletionError>() {
                Some(err) => match &err {
                    LanguageModelCompletionError::RateLimitExceeded { retry_after, .. }
                    | LanguageModelCompletionError::ServerOverloaded { retry_after, .. } => {
                        Some(retry_after.unwrap_or(Duration::from_secs(5)))
                    }
                    LanguageModelCompletionError::UpstreamProviderError {
                        status,
                        retry_after,
                        ..
                    } => {
                        let should_retry = matches!(
                            *status,
                            StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE
                        ) || status.as_u16() == 529;

                        if should_retry {
                            Some(retry_after.unwrap_or(Duration::from_secs(5)))
                        } else {
                            None
                        }
                    }
                    LanguageModelCompletionError::ApiReadResponseError { .. }
                    | LanguageModelCompletionError::ApiInternalServerError { .. }
                    | LanguageModelCompletionError::HttpSend { .. } => {
                        Some(Duration::from_secs(2_u64.pow((attempt - 1) as u32).min(30)))
                    }
                    _ => None,
                },
                _ => None,
            },
        };

        if let Some(retry_after) = retry_delay {
            let jitter = retry_after.mul_f64(rand::rng().random_range(0.0..1.0));
            eprintln!("Attempt #{attempt}: Retry after {retry_after:?} + jitter of {jitter:?}");
            #[allow(clippy::disallowed_methods)]
            async_io::Timer::after(retry_after + jitter).await;
        } else {
            return response;
        }
    }
}

#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_delete_function() {
    let input_file_path = "root/blame.rs";
    let input_file_content = include_str!("fixtures/delete_run_git_blame/before.rs");
    let output_file_content = include_str!("fixtures/delete_run_git_blame/after.rs");
    let possible_diffs = vec![
        language::unified_diff(input_file_content, output_file_content),
        language::unified_diff(
            input_file_content,
            &output_file_content
                .replace(
                    "const GIT_BLAME_NO_COMMIT_ERROR: &str = \"fatal: no such ref: HEAD\";\n",
                    "",
                )
                .replace(
                    "const GIT_BLAME_NO_PATH: &str = \"fatal: no such path\";\n",
                    "",
                ),
        ),
    ];

    eval_utils::eval(100, 0.95, eval_utils::NoProcessor, move || {
        run_eval(EvalInput::new(
            vec![
                message(
                    User,
                    [text(indoc::formatdoc! {"
                        Read the `{input_file_path}` file and delete `run_git_blame`. Just that
                        one function, not its usages.
                    "})],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_1",
                        ReadFileTool::NAME,
                        ReadFileToolInput {
                            path: input_file_path.into(),
                            start_line: None,
                            end_line: None,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_1",
                        ReadFileTool::NAME,
                        input_file_content,
                    )],
                ),
            ],
            input_file_path,
            Some(input_file_content.into()),
            EvalAssertion::assert_diff_any(possible_diffs.clone()),
        ))
    });
}

#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_extract_handle_command_output() {
    let input_file_path = "root/blame.rs";
    let input_file_content = include_str!("fixtures/extract_handle_command_output/before.rs");
    let possible_diffs = vec![
        include_str!("fixtures/extract_handle_command_output/possible-01.diff"),
        include_str!("fixtures/extract_handle_command_output/possible-02.diff"),
        include_str!("fixtures/extract_handle_command_output/possible-03.diff"),
        include_str!("fixtures/extract_handle_command_output/possible-04.diff"),
        include_str!("fixtures/extract_handle_command_output/possible-05.diff"),
        include_str!("fixtures/extract_handle_command_output/possible-06.diff"),
        include_str!("fixtures/extract_handle_command_output/possible-07.diff"),
        include_str!("fixtures/extract_handle_command_output/possible-08.diff"),
        include_str!("fixtures/extract_handle_command_output/possible-09.diff"),
    ];

    eval_utils::eval(100, 0.95, eval_utils::NoProcessor, move || {
        run_eval(EvalInput::new(
            vec![
                message(
                    User,
                    [text(indoc::formatdoc! {"
                        Read the `{input_file_path}` file and extract a method in
                        the final stanza of `run_git_blame` to deal with command failures,
                        call it `handle_command_output` and take the std::process::Output as the only parameter.
                        Do not document the method and do not add any comments.

                        Add it right next to `run_git_blame` and copy it verbatim from `run_git_blame`.
                    "})],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_1",
                        ReadFileTool::NAME,
                        ReadFileToolInput {
                            path: input_file_path.into(),
                            start_line: None,
                            end_line: None,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_1",
                        ReadFileTool::NAME,
                        input_file_content,
                    )],
                ),
            ],
            input_file_path,
            Some(input_file_content.into()),
            EvalAssertion::assert_diff_any(possible_diffs.clone()),
        ))
    });
}

#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_translate_doc_comments() {
    let input_file_path = "root/canvas.rs";
    let input_file_content = include_str!("fixtures/translate_doc_comments/before.rs");

    eval_utils::eval(200, 1., eval_utils::NoProcessor, move || {
        run_eval(EvalInput::new(
            vec![
                message(
                    User,
                    [text(indoc::formatdoc! {"
                        Read the `{input_file_path}` file and edit it (without overwriting it),
                        translating all the doc comments to italian.
                    "})],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_1",
                        ReadFileTool::NAME,
                        ReadFileToolInput {
                            path: input_file_path.into(),
                            start_line: None,
                            end_line: None,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_1",
                        ReadFileTool::NAME,
                        input_file_content,
                    )],
                ),
            ],
            input_file_path,
            Some(input_file_content.into()),
            EvalAssertion::judge_diff("Doc comments were translated to Italian"),
        ))
    });
}

#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_use_wasi_sdk_in_compile_parser_to_wasm() {
    let input_file_path = "root/lib.rs";
    let input_file_content =
        include_str!("fixtures/use_wasi_sdk_in_compile_parser_to_wasm/before.rs");

    eval_utils::eval(100, 0.95, eval_utils::NoProcessor, move || {
        run_eval(EvalInput::new(
            vec![
                message(
                    User,
                    [text(indoc::formatdoc! {"
                        Read the `{input_file_path}` file and change `compile_parser_to_wasm` to use `wasi-sdk` instead of emscripten.
                        Use `ureq` to download the SDK for the current platform and architecture.
                        Extract the archive into a sibling of `lib` inside the `tree-sitter` directory in the cache_dir.
                        Compile the parser to wasm using the `bin/clang` executable (or `bin/clang.exe` on windows)
                        that's inside of the archive.
                        Don't re-download the SDK if that executable already exists.

                        Use these clang flags: -fPIC -shared -Os -Wl,--export=tree_sitter_{{language_name}}

                        Here are the available wasi-sdk assets:
                        - wasi-sdk-25.0-x86_64-macos.tar.gz
                        - wasi-sdk-25.0-arm64-macos.tar.gz
                        - wasi-sdk-25.0-x86_64-linux.tar.gz
                        - wasi-sdk-25.0-arm64-linux.tar.gz
                        - wasi-sdk-25.0-x86_64-linux.tar.gz
                        - wasi-sdk-25.0-arm64-linux.tar.gz
                        - wasi-sdk-25.0-x86_64-windows.tar.gz
                    "})],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_1",
                        ReadFileTool::NAME,
                        ReadFileToolInput {
                            path: input_file_path.into(),
                            start_line: Some(971),
                            end_line: Some(1050),
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_1",
                        ReadFileTool::NAME,
                        lines(input_file_content, 971..1050),
                    )],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_2",
                        ReadFileTool::NAME,
                        ReadFileToolInput {
                            path: input_file_path.into(),
                            start_line: Some(1050),
                            end_line: Some(1100),
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_2",
                        ReadFileTool::NAME,
                        lines(input_file_content, 1050..1100),
                    )],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_3",
                        ReadFileTool::NAME,
                        ReadFileToolInput {
                            path: input_file_path.into(),
                            start_line: Some(1100),
                            end_line: Some(1150),
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_3",
                        ReadFileTool::NAME,
                        lines(input_file_content, 1100..1150),
                    )],
                ),
            ],
            input_file_path,
            Some(input_file_content.into()),
            EvalAssertion::judge_diff(indoc::indoc! {"
                    - The compile_parser_to_wasm method has been changed to use wasi-sdk
                    - ureq is used to download the SDK for current platform and architecture
                "}),
        ))
    });
}

#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_disable_cursor_blinking() {
    let input_file_path = "root/editor.rs";
    let input_file_content = include_str!("fixtures/disable_cursor_blinking/before.rs");
    let possible_diffs = vec![
        include_str!("fixtures/disable_cursor_blinking/possible-01.diff"),
        include_str!("fixtures/disable_cursor_blinking/possible-02.diff"),
        include_str!("fixtures/disable_cursor_blinking/possible-03.diff"),
        include_str!("fixtures/disable_cursor_blinking/possible-04.diff"),
    ];

    eval_utils::eval(100, 0.51, eval_utils::NoProcessor, move || {
        run_eval(EvalInput::new(
            vec![
                message(User, [text("Let's research how to cursor blinking works.")]),
                message(
                    Assistant,
                    [tool_use(
                        "tool_1",
                        GrepTool::NAME,
                        GrepToolInput {
                            regex: "blink".into(),
                            include_pattern: None,
                            offset: 0,
                            case_sensitive: false,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_1",
                        GrepTool::NAME,
                        [
                            lines(input_file_content, 100..400),
                            lines(input_file_content, 800..1300),
                            lines(input_file_content, 1600..2000),
                            lines(input_file_content, 5000..5500),
                            lines(input_file_content, 8000..9000),
                            lines(input_file_content, 18455..18470),
                            lines(input_file_content, 20000..20500),
                            lines(input_file_content, 21000..21300),
                        ]
                        .join("Match found:\n\n"),
                    )],
                ),
                message(
                    User,
                    [text(indoc::indoc! {"
                            Comment out the lines that interact with the BlinkManager.
                            Keep the outer `update` blocks, but comments everything that's inside (including if statements).
                            Don't add additional comments.
                        "})],
                ),
            ],
            input_file_path,
            Some(input_file_content.into()),
            EvalAssertion::assert_diff_any(possible_diffs.clone()),
        ))
    });
}

#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_from_pixels_constructor() {
    let input_file_path = "root/canvas.rs";
    let input_file_content = include_str!("fixtures/from_pixels_constructor/before.rs");

    eval_utils::eval(100, 0.95, eval_utils::NoProcessor, move || {
        run_eval(EvalInput::new(
            vec![
                message(
                    User,
                    [text(indoc::indoc! {"
                            Introduce a new `from_pixels` constructor in Canvas and
                            also add tests for it in the same file.
                        "})],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_1",
                        ReadFileTool::NAME,
                        ReadFileToolInput {
                            path: input_file_path.into(),
                            start_line: None,
                            end_line: None,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_1",
                        ReadFileTool::NAME,
                        input_file_content,
                    )],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_2",
                        GrepTool::NAME,
                        GrepToolInput {
                            regex: "mod\\s+tests".into(),
                            include_pattern: Some("font-kit/src/canvas.rs".into()),
                            offset: 0,
                            case_sensitive: false,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result("tool_2", GrepTool::NAME, "No matches found")],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_3",
                        GrepTool::NAME,
                        GrepToolInput {
                            regex: "mod\\s+tests".into(),
                            include_pattern: Some("font-kit/src/**/*.rs".into()),
                            offset: 0,
                            case_sensitive: false,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result("tool_3", GrepTool::NAME, "No matches found")],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_4",
                        GrepTool::NAME,
                        GrepToolInput {
                            regex: "#\\[test\\]".into(),
                            include_pattern: Some("font-kit/src/**/*.rs".into()),
                            offset: 0,
                            case_sensitive: false,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_4",
                        GrepTool::NAME,
                        indoc::indoc! {"
                                Found 6 matches:

                                ## Matches in font-kit/src/loaders/core_text.rs

                                ### mod test › L926-936
                                ```
                                mod test {
                                    use super::Font;
                                    use crate::properties::{Stretch, Weight};

                                    #[cfg(feature = \"source\")]
                                    use crate::source::SystemSource;

                                    static TEST_FONT_POSTSCRIPT_NAME: &'static str = \"ArialMT\";

                                    #[cfg(feature = \"source\")]
                                    #[test]
                                ```

                                55 lines remaining in ancestor node. Read the file to see all.

                                ### mod test › L947-951
                                ```
                                    }

                                    #[test]
                                    fn test_core_text_to_css_font_weight() {
                                        // Exact matches
                                ```

                                ### mod test › L959-963
                                ```
                                    }

                                    #[test]
                                    fn test_core_text_to_css_font_stretch() {
                                        // Exact matches
                                ```

                                ## Matches in font-kit/src/loaders/freetype.rs

                                ### mod test › L1238-1248
                                ```
                                mod test {
                                    use crate::loaders::freetype::Font;

                                    static PCF_FONT_PATH: &str = \"resources/tests/times-roman-pcf/timR12.pcf\";
                                    static PCF_FONT_POSTSCRIPT_NAME: &str = \"Times-Roman\";

                                    #[test]
                                    fn get_pcf_postscript_name() {
                                        let font = Font::from_path(PCF_FONT_PATH, 0).unwrap();
                                        assert_eq!(font.postscript_name().unwrap(), PCF_FONT_POSTSCRIPT_NAME);
                                    }
                                ```

                                1 lines remaining in ancestor node. Read the file to see all.

                                ## Matches in font-kit/src/sources/core_text.rs

                                ### mod test › L265-275
                                ```
                                mod test {
                                    use crate::properties::{Stretch, Weight};

                                    #[test]
                                    fn test_css_to_core_text_font_weight() {
                                        // Exact matches
                                        assert_eq!(super::css_to_core_text_font_weight(Weight(100.0)), -0.7);
                                        assert_eq!(super::css_to_core_text_font_weight(Weight(400.0)), 0.0);
                                        assert_eq!(super::css_to_core_text_font_weight(Weight(700.0)), 0.4);
                                        assert_eq!(super::css_to_core_text_font_weight(Weight(900.0)), 0.8);

                                ```

                                27 lines remaining in ancestor node. Read the file to see all.

                                ### mod test › L278-282
                                ```
                                    }

                                    #[test]
                                    fn test_css_to_core_text_font_stretch() {
                                        // Exact matches
                                ```
                            "},
                    )],
                ),
            ],
            input_file_path,
            Some(input_file_content.into()),
            EvalAssertion::judge_diff(indoc::indoc! {"
                        - The diff contains a new `from_pixels` constructor
                        - The diff contains new tests for the `from_pixels` constructor
                    "}),
        ))
    });
}

#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_zode() {
    let input_file_path = "root/zode.py";
    let input_content = None;

    eval_utils::eval(50, 1., eval_utils::NoProcessor, move || {
        run_eval(EvalInput::new(
            vec![
                message(User, [text(include_str!("fixtures/zode/prompt.md"))]),
                message(
                    Assistant,
                    [
                        tool_use(
                            "tool_1",
                            ReadFileTool::NAME,
                            ReadFileToolInput {
                                path: "root/eval/react.py".into(),
                                start_line: None,
                                end_line: None,
                            },
                        ),
                        tool_use(
                            "tool_2",
                            ReadFileTool::NAME,
                            ReadFileToolInput {
                                path: "root/eval/react_test.py".into(),
                                start_line: None,
                                end_line: None,
                            },
                        ),
                    ],
                ),
                message(
                    User,
                    [
                        tool_result(
                            "tool_1",
                            ReadFileTool::NAME,
                            include_str!("fixtures/zode/react.py"),
                        ),
                        tool_result(
                            "tool_2",
                            ReadFileTool::NAME,
                            include_str!("fixtures/zode/react_test.py"),
                        ),
                    ],
                ),
            ],
            input_file_path,
            input_content.clone(),
            EvalAssertion::new(async move |sample, _, _cx| {
                let invalid_starts = [' ', '`', '\n'];
                let mut message = String::new();
                for start in invalid_starts {
                    if sample.text_after.starts_with(start) {
                        message.push_str(&format!("The sample starts with a {:?}\n", start));
                        break;
                    }
                }
                message.pop();

                if message.is_empty() {
                    Ok(EvalAssertionOutcome {
                        score: 100,
                        message: None,
                    })
                } else {
                    Ok(EvalAssertionOutcome {
                        score: 0,
                        message: Some(message),
                    })
                }
            }),
        ))
    });
}

#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_add_overwrite_test() {
    let input_file_path = "root/action_log.rs";
    let input_file_content = include_str!("fixtures/add_overwrite_test/before.rs");

    eval_utils::eval(200, 0.5, eval_utils::NoProcessor, move || {
        run_eval(EvalInput::new(
            vec![
                message(
                    User,
                    [text(indoc::indoc! {"
                            Introduce a new test in `action_log.rs` to test overwriting a file.
                            That is, a file already exists, but we call `buffer_created` as if the file were new.
                            Take inspiration from all the other tests in the file.
                        "})],
                ),
                message(
                    Assistant,
                    [tool_use(
                        "tool_1",
                        ReadFileTool::NAME,
                        ReadFileToolInput {
                            path: input_file_path.into(),
                            start_line: None,
                            end_line: None,
                        },
                    )],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_1",
                        ReadFileTool::NAME,
                        indoc::indoc! {"
                                pub struct ActionLog [L13-20]
                                 tracked_buffers [L15]
                                 edited_since_project_diagnostics_check [L17]
                                 project [L19]
                                impl ActionLog [L22-498]
                                 pub fn new [L24-30]
                                 pub fn project [L32-34]
                                 pub fn checked_project_diagnostics [L37-39]
                                 pub fn has_edited_files_since_project_diagnostics_check [L42-44]
                                 fn track_buffer_internal [L46-101]
                                 fn handle_buffer_event [L103-116]
                                 fn handle_buffer_edited [L118-123]
                                 fn handle_buffer_file_changed [L125-158]
                                 async fn maintain_diff [L160-264]
                                 pub fn buffer_read [L267-269]
                                 pub fn buffer_created [L272-276]
                                 pub fn buffer_edited [L279-287]
                                 pub fn will_delete_buffer [L289-304]
                                 pub fn keep_edits_in_range [L306-364]
                                 pub fn reject_edits_in_ranges [L366-459]
                                 pub fn keep_all_edits [L461-473]
                                 pub fn changed_buffers [L476-482]
                                 pub fn stale_buffers [L485-497]
                                fn apply_non_conflicting_edits [L500-561]
                                fn diff_snapshots [L563-585]
                                fn point_to_row_edit [L587-614]
                                enum ChangeAuthor [L617-620]
                                 User [L618]
                                 Agent [L619]
                                enum TrackedBufferStatus [L623-627]
                                 Created [L624]
                                 Modified [L625]
                                 Deleted [L626]
                                struct TrackedBuffer [L629-641]
                                 buffer [L630]
                                 base_text [L631]
                                 unreviewed_changes [L632]
                                 status [L633]
                                 version [L634]
                                 diff [L635]
                                 snapshot [L636]
                                 diff_update [L637]
                                 _open_lsp_handle [L638]
                                 _maintain_diff [L639]
                                 _subscription [L640]
                                impl TrackedBuffer [L643-657]
                                 fn has_changes [L644-650]
                                 fn schedule_diff_update [L652-656]
                                pub struct ChangedBuffer [L659-661]
                                 pub diff [L660]
                                mod tests [L664-1574]
                                 fn init_logger [L678-682]
                                 fn init_test [L684-691]
                                 async fn test_keep_edits [L694-769]
                                 async fn test_deletions [L772-854]
                                 async fn test_overlapping_user_edits [L857-951]
                                 async fn test_creating_files [L954-1010]
                                 async fn test_deleting_files [L1013-1120]
                                 async fn test_reject_edits [L1123-1255]
                                 async fn test_reject_multiple_edits [L1258-1331]
                                 async fn test_reject_deleted_file [L1334-1388]
                                 async fn test_reject_created_file [L1391-1443]
                                 async fn test_random_diffs [L1446-1535]
                                  fn quiesce [L1510-1534]
                                 struct HunkStatus [L1538-1542]
                                  range [L1539]
                                  diff_status [L1540]
                                  old_text [L1541]
                                 fn unreviewed_hunks [L1544-1573]

                                Showing symbols 1-69 (total symbols: 69)

                                Using the line numbers in this outline, you can call this tool again while specifying
                                the start_line and end_line fields to see the implementations of symbols in the outline.
                            "},
                    )],
                ),
                message(
                    Assistant,
                    [
                        text(
                            "Let's look at the existing test cases that are related to file creation to understand the pattern:",
                        ),
                        tool_use(
                            "tool_2",
                            ReadFileTool::NAME,
                            ReadFileToolInput {
                                path: input_file_path.into(),
                                start_line: Some(953),
                                end_line: Some(1010),
                            },
                        ),
                    ],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_2",
                        ReadFileTool::NAME,
                        lines(input_file_content, 953..1010),
                    )],
                ),
                message(
                    Assistant,
                    [
                        text(
                            "Now let's look at another relevant test to understand how they're structured:",
                        ),
                        tool_use(
                            "tool_3",
                            ReadFileTool::NAME,
                            ReadFileToolInput {
                                path: input_file_path.into(),
                                start_line: Some(1012),
                                end_line: Some(1120),
                            },
                        ),
                    ],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_3",
                        ReadFileTool::NAME,
                        lines(input_file_content, 1012..1120),
                    )],
                ),
                message(
                    Assistant,
                    [
                        text("Now let's look at how `buffer_created` is implemented:"),
                        tool_use(
                            "tool_4",
                            ReadFileTool::NAME,
                            ReadFileToolInput {
                                path: input_file_path.into(),
                                start_line: Some(271),
                                end_line: Some(276),
                            },
                        ),
                    ],
                ),
                message(
                    User,
                    [tool_result(
                        "tool_4",
                        ReadFileTool::NAME,
                        lines(input_file_content, 271..276),
                    )],
                ),
            ],
            input_file_path,
            Some(input_file_content.into()),
            EvalAssertion::judge_diff(
                "A new test for overwritten files was created, without changing any previous test",
            ),
        ))
    });
}
