[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"me":3,"catalog:en:ai-engineer\u002Fevaluation-testing":4,"config":192},null,{"field_key":5,"field_name":6,"seniority":7,"topic_key":8,"topic_name":9,"spec_key":7,"spec_name":7,"locale":10,"cell_total":11,"field_total":12,"seniorities":13,"topics":17,"specs":97,"samples":108},"ai-engineer","AI Engineer","","evaluation-testing","Evaluation Testing","en",75,2025,[14,15,16],"junior","mid","senior",[18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,61,64,67,70,73,76,79,82,85,88,91,94],{"key":19,"name":20,"count":11},"agents-tool-use","Agents Tool Use",{"key":22,"name":23,"count":11},"aimlops-deployment-strategies-ml","Aimlops Deployment Strategies Ml",{"key":25,"name":26,"count":11},"aimlops-experiment-tracking-reproducibility","Aimlops Experiment Tracking Reproducibility",{"key":28,"name":29,"count":11},"aimlops-feature-store-data-versioning","Aimlops Feature Store Data Versioning",{"key":31,"name":32,"count":11},"aimlops-ml-cicd-pipelines","Aimlops Ml Cicd Pipelines",{"key":34,"name":35,"count":11},"aimlops-model-monitoring-drift-detection","Aimlops Model Monitoring Drift Detection",{"key":37,"name":38,"count":11},"aimlops-model-versioning-registry","Aimlops Model Versioning Registry",{"key":40,"name":41,"count":11},"aipy-async-concurrency-ml-serving","Aipy Async Concurrency Ml Serving",{"key":43,"name":44,"count":11},"aipy-data-pipeline-dataloader","Aipy Data Pipeline Dataloader",{"key":46,"name":47,"count":11},"aipy-gpu-memory-management","Aipy Gpu Memory Management",{"key":49,"name":50,"count":11},"aipy-numpy-vectorization-broadcasting","Aipy Numpy Vectorization Broadcasting",{"key":52,"name":53,"count":11},"aipy-python-ml-packaging-environments","Aipy Python Ml Packaging Environments",{"key":55,"name":56,"count":11},"aipy-tensor-ops-autograd","Aipy Tensor Ops Autograd",{"key":58,"name":59,"count":11},"embeddings-vector-search","Embeddings Vector Search",{"key":8,"name":9,"count":11},{"key":62,"name":63,"count":11},"fine-tuning-adaptation","Fine Tuning Adaptation",{"key":65,"name":66,"count":11},"inference-serving","Inference Serving",{"key":68,"name":69,"count":11},"llm-fundamentals","Llm Fundamentals",{"key":71,"name":72,"count":11},"prompt-engineering","Prompt Engineering",{"key":74,"name":75,"count":11},"rag-chunking-indexing","Rag Chunking Indexing",{"key":77,"name":78,"count":11},"rag-embeddings-similarity","Rag Embeddings Similarity",{"key":80,"name":81,"count":11},"rag-evaluation","Rag Evaluation",{"key":83,"name":84,"count":11},"rag-generation-context","Rag Generation Context",{"key":86,"name":87,"count":11},"rag-reranking-fusion","Rag Reranking Fusion",{"key":89,"name":90,"count":11},"rag-retrieval","Rag Retrieval",{"key":92,"name":93,"count":11},"rag-retrieval-search","Rag Retrieval Search",{"key":95,"name":96,"count":11},"safety-guardrails","Safety Guardrails",[98,102,105],{"key":99,"name":100,"count":101},"llm-rag","LLM\u002FRAG",450,{"key":103,"name":104,"count":101},"mlops","MLOps",{"key":106,"name":107,"count":101},"python","Python",[109,127,140,153,166,179],{"id":110,"topic":9,"difficulty":111,"body":112,"options":113,"correct_key":121,"explanation":126},"019f5912-45a8-7984-bdab-858774e23009",1,"You ask an LLM the same question twice with temperature > 0 and get two differently worded but both reasonable answers. Why does this make evaluating LLM output harder than evaluating traditional software?",[114,117,120,123],{"key":115,"text":116},"a","Because non-determinism only affects creative-writing tasks, so factual question-answering is essentially unaffected by it",{"key":118,"text":119},"b","Because LLMs always produce the exact same output for a fixed prompt, so evaluation is actually simpler than for traditional software",{"key":121,"text":122},"c","Because a single run's output can't reliably represent the model's typical quality across attempts",{"key":124,"text":125},"d","Because fixing the random seed once makes output fully deterministic afterward across every model and provider","Since output varies run to run, one sample isn't representative of typical quality (c) — evaluation needs multiple samples or judge-based scoring tolerant of paraphrase. Non-determinism affects factual QA too, not just creative tasks (a is wrong). LLMs at temperature > 0 don't produce identical output each call, so it isn't simpler for that reason (b is wrong). Seed handling isn't guaranteed to fully eliminate variance across every provider or model (d is wrong).",{"id":128,"topic":9,"difficulty":111,"body":129,"options":130,"correct_key":115,"explanation":139},"019f5912-45a9-739c-81c3-ede1ecd60e51","In the context of evaluating an LLM application, what is a 'golden set' (eval set)?",[131,133,135,137],{"key":115,"text":132},"A fixed collection of representative inputs paired with expected outputs, used to measure quality consistently",{"key":118,"text":134},"A set of the highest-scoring production requests, automatically saved whenever user feedback is positive",{"key":121,"text":136},"A list of banned words and phrases that a content filter checks every response against before returning it",{"key":124,"text":138},"A random sample of live traffic pulled fresh right before every deployment so the test data stays current","A golden set is a curated, fixed benchmark with known expected outputs, reused across changes so scores are comparable over time (a). Automatically saved positive-feedback requests (b) describe an implicit feedback log, not a curated eval set. A banned-word list (c) is a guardrail\u002Ffilter artifact, not an evaluation benchmark. Pulling fresh random traffic each time (d) removes the fixed baseline needed for consistent before\u002Fafter comparison.",{"id":141,"topic":9,"difficulty":111,"body":142,"options":143,"correct_key":124,"explanation":152},"019f5912-45aa-737d-98b3-490290c777f0","For a task where a model extracts a single structured value (e.g., an order ID) from a document, what is a key limitation of scoring outputs with 'exact match'?",[144,146,148,150],{"key":115,"text":145},"It only works for outputs written in English and produces errors on any other language",{"key":118,"text":147},"It cannot be computed without first sending the output to a second LLM for review",{"key":121,"text":149},"It requires the golden set to contain at least a million examples before it produces a meaningful score",{"key":124,"text":151},"It scores 0 for any output that differs in formatting (e.g., extra whitespace or leading zeros), even if the value is correct","Exact match is brittle to superficial formatting differences even when the value is functionally correct, so it often needs normalization or looser matching (d). Exact match is a plain string comparison, so it's language-agnostic (a is wrong) and needs no second LLM call (b is wrong). It also works fine on small golden sets; there's no minimum-example requirement (c is wrong).",{"id":154,"topic":9,"difficulty":111,"body":155,"options":156,"correct_key":118,"explanation":165},"019f5912-45b0-72e0-9c29-0e60e77be468","What is the main practical difference between 'offline evaluation' and 'production monitoring' for an LLM application?",[157,159,161,163],{"key":115,"text":158},"Offline evaluation is always performed by a human, while production monitoring is always fully automated",{"key":118,"text":160},"Offline evaluation runs against a fixed set of test cases before shipping; production monitoring observes live traffic after release",{"key":121,"text":162},"Offline evaluation only checks latency and cost, while production monitoring only checks correctness",{"key":124,"text":164},"Offline evaluation becomes unnecessary once production monitoring is set up, since live traffic eventually covers every case offline testing would","The core distinction is timing and data source: offline eval uses a fixed pre-release test set, production monitoring watches live traffic continuously after release (b). Neither is restricted to human-only or fully-automated by definition (a is wrong). Both can track correctness, latency, or cost depending on setup (c is wrong). Production monitoring doesn't guarantee coverage of the specific edge cases offline testing deliberately targets, so offline eval remains useful (d is wrong).",{"id":167,"topic":9,"difficulty":111,"body":168,"options":169,"correct_key":115,"explanation":178},"019f5912-45b0-7eb2-bdbd-2894f3758d0e","What does the term 'LLM-as-judge' refer to in evaluation of LLM applications?",[170,172,174,176],{"key":115,"text":171},"Using a separate LLM call, guided by a rubric or reference answer, to score the system's outputs",{"key":118,"text":173},"A legal requirement that a model vendor's own model must certify the safety of any application built on top of it",{"key":121,"text":175},"A technique where the model being evaluated is asked to guess its own accuracy percentage in a single pass",{"key":124,"text":177},"A metric that counts how many times a user had to rephrase their prompt before getting a satisfactory answer","LLM-as-judge means using a (typically separate) LLM call guided by criteria to score or rank outputs (a). There is no such vendor-certification requirement (b is fabricated). Self-reported confidence (c) is a different, generally unreliable technique, not judging another output against criteria. Counting rephrase attempts (d) describes a distinct usage metric, unrelated to judging output quality.",{"id":180,"topic":9,"difficulty":111,"body":181,"options":182,"correct_key":121,"explanation":191},"019f5912-45b1-7808-a298-9cc36f0b8e44","After changing a prompt template used in production, why is it good practice to re-run a suite of known test cases before deploying the change?",[183,185,187,189],{"key":115,"text":184},"Because prompt changes are not allowed to be deployed without written sign-off from the model provider",{"key":118,"text":186},"Because re-running the test cases automatically reverts the prompt to the previous version if any case fails",{"key":121,"text":188},"To catch cases where previously correct behavior has regressed, even if the change targeted something else",{"key":124,"text":190},"To reduce the number of tokens the new prompt consumes compared to the old one","Regression testing exists to catch unintended breakage in previously-working behavior, independent of what the new change was meant to fix (c). There's no such provider sign-off requirement (a is fabricated, provider-agnostic violation). Running a suite only observes results; it doesn't itself revert anything without separate tooling (b is wrong). Token count is a separate cost concern, not the purpose of regression testing (d is wrong).",{"fields":193,"seniorities":372,"interview_shapes":373,"locales":378,"oauth":380,"question_count":383,"coach_enabled":384,"jd_match_enabled":384},[194,217,238,255,279,286,305,324,346,353,359,366],{"key":195,"name_tr":196,"name_en":196,"sort":111,"specializations":197},"backend","Backend",[198,201,204,205,208,211,214],{"key":199,"name":200,"field":195},"general","Genel",{"key":202,"name":203,"field":195},"go","Go",{"key":106,"name":107,"field":195},{"key":206,"name":207,"field":195},"java","Java",{"key":209,"name":210,"field":195},"csharp","C#\u002F.NET",{"key":212,"name":213,"field":195},"nodejs","Node.js",{"key":215,"name":216,"field":195},"php","PHP",{"key":218,"name_tr":219,"name_en":219,"sort":220,"specializations":221},"frontend","Frontend",2,[222,223,226,229,232,235],{"key":199,"name":200,"field":218},{"key":224,"name":225,"field":218},"javascript","JavaScript",{"key":227,"name":228,"field":218},"typescript","TypeScript",{"key":230,"name":231,"field":218},"react","React",{"key":233,"name":234,"field":218},"vue","Vue",{"key":236,"name":237,"field":218},"angular","Angular",{"key":239,"name_tr":240,"name_en":240,"sort":241,"specializations":242},"fullstack","Fullstack",3,[243,244,245,246,247,248,249,250,251,252,253,254],{"key":199,"name":200,"field":239},{"key":202,"name":203,"field":195},{"key":106,"name":107,"field":195},{"key":206,"name":207,"field":195},{"key":209,"name":210,"field":195},{"key":212,"name":213,"field":195},{"key":215,"name":216,"field":195},{"key":224,"name":225,"field":218},{"key":227,"name":228,"field":218},{"key":230,"name":231,"field":218},{"key":233,"name":234,"field":218},{"key":236,"name":237,"field":218},{"key":256,"name_tr":257,"name_en":257,"sort":258,"specializations":259},"devops-cloud","DevOps \u002F Cloud",4,[260,261,264,267,270,273,276],{"key":199,"name":200,"field":256},{"key":262,"name":263,"field":256},"aws","AWS",{"key":265,"name":266,"field":256},"gcp","GCP",{"key":268,"name":269,"field":256},"azure","Azure",{"key":271,"name":272,"field":256},"kubernetes","Kubernetes",{"key":274,"name":275,"field":256},"terraform","Terraform",{"key":277,"name":278,"field":256},"linux","Linux",{"key":5,"name_tr":6,"name_en":6,"sort":280,"specializations":281},5,[282,283,284,285],{"key":199,"name":200,"field":5},{"key":106,"name":107,"field":5},{"key":99,"name":100,"field":5},{"key":103,"name":104,"field":5},{"key":287,"name_tr":288,"name_en":289,"sort":290,"specializations":291},"database","Veritabanı","Database",6,[292,293,296,299,302],{"key":199,"name":200,"field":287},{"key":294,"name":295,"field":287},"postgresql","PostgreSQL",{"key":297,"name":298,"field":287},"mysql","MySQL",{"key":300,"name":301,"field":287},"mongodb","MongoDB",{"key":303,"name":304,"field":287},"redis","Redis",{"key":306,"name_tr":307,"name_en":308,"sort":309,"specializations":310},"mobile","Mobil","Mobile",7,[311,312,315,318,321],{"key":199,"name":200,"field":306},{"key":313,"name":314,"field":306},"ios-swift","iOS (Swift)",{"key":316,"name":317,"field":306},"android-kotlin","Android (Kotlin)",{"key":319,"name":320,"field":306},"flutter","Flutter",{"key":322,"name":323,"field":306},"react-native","React Native",{"key":325,"name_tr":326,"name_en":327,"sort":328,"specializations":329},"security","Güvenlik","Security",8,[330,331,334,337,340,343],{"key":199,"name":200,"field":325},{"key":332,"name":333,"field":325},"appsec","AppSec",{"key":335,"name":336,"field":325},"offensive-pentest","Offensive \u002F Pentest",{"key":338,"name":339,"field":325},"cloud-security","Cloud Security",{"key":341,"name":342,"field":325},"devsecops","DevSecOps",{"key":344,"name":345,"field":325},"blue-team-incident","Blue Team \u002F Incident",{"key":347,"name_tr":348,"name_en":349,"sort":350,"specializations":351},"qa-test-automation","QA \u002F Test Otomasyonu","QA \u002F Test Automation",9,[352],{"key":199,"name":200,"field":347},{"key":354,"name_tr":355,"name_en":355,"sort":356,"specializations":357},"data-engineer","Data Engineer",10,[358],{"key":199,"name":200,"field":354},{"key":360,"name_tr":361,"name_en":362,"sort":363,"specializations":364},"game-dev","Oyun Geliştirme","Game Development",11,[365],{"key":199,"name":200,"field":360},{"key":367,"name_tr":368,"name_en":368,"sort":369,"specializations":370},"ml-engineer","ML Engineer",12,[371],{"key":199,"name":200,"field":367},[14,15,16],{"junior":374,"mid":376,"senior":377},{"questions":375,"median_sec":3},20,{"questions":375,"median_sec":3},{"questions":375,"median_sec":3},[379,10],"tr",[381,382],"google","github",21750,true]