[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"me":3,"catalog:en:ml-engineer\u002Fti-distributed-strategies-sync":4,"config":212},null,{"field_key":5,"field_name":6,"seniority":7,"topic_key":8,"topic_name":9,"spec_key":7,"spec_name":7,"locale":10,"cell_total":11,"field_total":12,"seniorities":13,"topics":17,"specs":112,"samples":126},"ml-engineer","ML Engineer","","ti-distributed-strategies-sync","Ti Distributed Strategies Sync","en",75,2400,[14,15,16],"junior","mid","senior",[18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,103,106,109],{"key":19,"name":20,"count":11},"cml-distance-clustering-dimreduction","Cml Distance Clustering Dimreduction",{"key":22,"name":23,"count":11},"cml-gradient-boosting-mechanics","Cml Gradient Boosting Mechanics",{"key":25,"name":26,"count":11},"cml-linear-logistic-internals","Cml Linear Logistic Internals",{"key":28,"name":29,"count":11},"cml-probabilistic-models-calibration","Cml Probabilistic Models Calibration",{"key":31,"name":32,"count":11},"cml-svm-kernels-margins","Cml Svm Kernels Margins",{"key":34,"name":35,"count":11},"cml-trees-randomforest-internals","Cml Trees Randomforest Internals",{"key":37,"name":38,"count":11},"dl-architecture-layers","Dl Architecture Layers",{"key":40,"name":41,"count":11},"dl-initialization-optimizers","Dl Initialization Optimizers",{"key":43,"name":44,"count":11},"dl-losses-output-layers","Dl Losses Output Layers",{"key":46,"name":47,"count":11},"dl-normalization-regularization","Dl Normalization Regularization",{"key":49,"name":50,"count":11},"dl-sequence-attention","Dl Sequence Attention",{"key":52,"name":53,"count":11},"dl-training-dynamics-backprop","Dl Training Dynamics Backprop",{"key":55,"name":56,"count":11},"fe-binning-discretization","Fe Binning Discretization",{"key":58,"name":59,"count":11},"fe-categorical-encoding-mechanics","Fe Categorical Encoding Mechanics",{"key":61,"name":62,"count":11},"fe-interactions-basis-expansion","Fe Interactions Basis Expansion",{"key":64,"name":65,"count":11},"fe-numeric-transforms-distributions","Fe Numeric Transforms Distributions",{"key":67,"name":68,"count":11},"fe-temporal-cyclical-features","Fe Temporal Cyclical Features",{"key":70,"name":71,"count":11},"fe-text-highcardinality-features","Fe Text Highcardinality Features",{"key":73,"name":74,"count":11},"ml-data-preparation","Ml Data Preparation",{"key":76,"name":77,"count":11},"ml-deployment-serving","Ml Deployment Serving",{"key":79,"name":80,"count":11},"ml-experimentation-reproducibility","Ml Experimentation Reproducibility",{"key":82,"name":83,"count":11},"ml-model-selection-tuning","Ml Model Selection Tuning",{"key":85,"name":86,"count":11},"ml-model-training-evaluation","Ml Model Training Evaluation",{"key":88,"name":89,"count":11},"ml-monitoring-drift","Ml Monitoring Drift",{"key":91,"name":92,"count":11},"ml-problem-framing","Ml Problem Framing",{"key":94,"name":95,"count":11},"ml-scaling-performance","Ml Scaling Performance",{"key":97,"name":98,"count":11},"ti-checkpointing-fault-tolerance","Ti Checkpointing Fault Tolerance",{"key":100,"name":101,"count":11},"ti-cluster-scheduling-resources","Ti Cluster Scheduling Resources",{"key":8,"name":9,"count":11},{"key":104,"name":105,"count":11},"ti-input-pipeline-throughput","Ti Input Pipeline Throughput",{"key":107,"name":108,"count":11},"ti-throughput-profiling-bottlenecks","Ti Throughput Profiling Bottlenecks",{"key":110,"name":111,"count":11},"ti-training-determinism-numerics","Ti Training Determinism Numerics",[113,117,120,123],{"key":114,"name":115,"count":116},"classical-ml","Classical ML",450,{"key":118,"name":119,"count":116},"deep-learning","Deep Learning",{"key":121,"name":122,"count":116},"feature-engineering","Feature Engineering",{"key":124,"name":125,"count":116},"training-infrastructure","Training Infrastructure",[127,145,159,173,186,199],{"id":128,"topic":9,"difficulty":129,"body":130,"options":131,"correct_key":139,"explanation":144},"01a05d13-8d21-739a-baa2-7dab67be4093",3,"You run ring all-reduce to sum gradients across 8 ranks. Each rank's local gradient tensor is 800 MB. Roughly how much data does EACH rank send over the network in total during the ring all-reduce, ignoring protocol overhead?",[132,135,138,141],{"key":133,"text":134},"a","800 MB, since the tensor is forwarded just once per rank.",{"key":136,"text":137},"b","6400 MB, a full copy received from all 7 other ranks.",{"key":139,"text":140},"c","About 1400 MB, since the 2(N-1)\u002FN factor applies here.",{"key":142,"text":143},"d","100 MB: no chunk is ever re-sent.","Ring all-reduce moves the tensor around the ring in chunk-sized pieces over two phases, so each rank ends up sending and receiving roughly (N-1)\u002FN of the tensor twice, not a handful of full copies; total traffic per rank is about 2(N-1)\u002FN * size. With N=8 and 800 MB that is 2*7\u002F8*800 ≈ 1400 MB — far less than sending 7 full copies.",{"id":146,"topic":9,"difficulty":147,"body":148,"options":149,"correct_key":136,"explanation":158},"01a05d13-8d23-706e-b02c-e3fb673e72d7",1,"In ring all-reduce with N ranks, how many total communication steps (reduce-scatter + all-gather) does the algorithm run, independent of tensor size?",[150,152,154,156],{"key":133,"text":151},"N steps, one per rank.",{"key":136,"text":153},"2(N-1) steps: reduce-scatter, then all-gather.",{"key":139,"text":155},"log2(N) steps, like a tree reduction.",{"key":142,"text":157},"1 step, a single collective call.","Ring all-reduce is built from two phases, each requiring N-1 point-to-point exchanges around the ring: a reduce-scatter phase that leaves each rank with one reduced chunk, then an all-gather phase that distributes every chunk to every rank. That is 2(N-1) steps total, not log2(N) — the log2(N) step count belongs to tree\u002Fhalving-doubling style reductions instead.",{"id":160,"topic":9,"difficulty":161,"body":162,"options":163,"correct_key":133,"explanation":172},"01a05d13-8d25-7917-8a8c-e6404a3cd40e",2,"Two ranks run this DDP training step with gloo backend (PyTorch 2.8):\n```python\ntorch.manual_seed(0)\nm = nn.Linear(2, 1, bias=False)\nddp = DDP(m)\nddp(torch.full((1, 2), float(rank + 1))).sum().backward()\n```\nRank 0's local (unsynced) gradient would be [1, 1] and rank 1's would be [2, 2]. What does `m.weight.grad` actually hold on both ranks after `backward()` returns?",[164,166,168,170],{"key":133,"text":165},"[1.5, 1.5] on both ranks, the averaged value.",{"key":136,"text":167},"[3.0, 3.0] on both ranks, the summed value.",{"key":139,"text":169},"[1.0, 1.0] on rank 0 and [2.0, 2.0] on rank 1, unsynced.",{"key":142,"text":171},"[1.5, 1.5] on rank 0 only; rank 1 keeps [2.0, 2.0].","This was measured directly: `dist.all_reduce(t, op=SUM)` on the same values gives 3.0, but DDP's built-in backward hook divides the summed gradient by `world_size` before writing it into `.grad`, so both ranks end up with the average [1.5, 1.5], not the sum, and both ranks see the identical value after `backward()`.",{"id":174,"topic":9,"difficulty":129,"body":175,"options":176,"correct_key":142,"explanation":185},"01a05d13-8d28-72d3-9673-bfd9e2cee01b","A team migrates from single-GPU training to 4-rank DDP. To keep the code 'simple', they replace DDP's comm hook with a custom one that calls `dist.all_reduce(bucket.buffer(), op=SUM)` directly, without dividing by world size (this was tested: doing so leaves `.grad` at the raw cross-rank SUM, not the average). Keeping the same learning rate as the single-GPU run, what is the most likely immediate symptom?",[177,179,181,183],{"key":133,"text":178},"No change; PyTorch auto-rescales the optimizer's LR.",{"key":136,"text":180},"Training becomes bit-deterministic across runs.",{"key":139,"text":182},"The loss curve is identical to the single-GPU run.",{"key":142,"text":184},"Effective step size is roughly 4x too large for the LR.","An update is `w -= lr * grad`. If `grad` is now the sum across 4 ranks instead of the average, it is roughly 4x the magnitude a correctly-scaled gradient would have (for similar per-rank gradients), so the effective step size scales up by about world_size — commonly seen as instability or divergence unless the learning rate is divided by world_size to compensate.",{"id":187,"topic":9,"difficulty":161,"body":188,"options":189,"correct_key":142,"explanation":198},"01a05d13-8d2a-763c-8522-24a50afdc64c","Rank 0 processes a local batch of 1 sample and rank 1 processes a local batch of 3 samples in the same DDP step (an uneven split). This was measured: DDP's synced gradient equals the FLAT average of each rank's local per-sample-mean gradient (1\u002Fworld_size), not a sample-count-weighted average across all 4 samples combined. What does this imply?",[190,192,194,196],{"key":133,"text":191},"Nothing in practice; DDP re-weights by local batch size first.",{"key":136,"text":193},"The synced gradient is undefined (NaN) when batch sizes differ.",{"key":139,"text":195},"Rank 0's sample is silently dropped from the average.",{"key":142,"text":197},"Rank 1's 3 samples are under-weighted vs a true pooled mean.","A true pooled mean over the 1+3=4 samples would weight rank 1's contribution 3x more than rank 0's. But DDP's flat 1\u002Fworld_size average gives rank 0's local mean (over 1 sample) the same weight as rank 1's local mean (over 3 samples), so rank 1's samples end up under-represented relative to a sample-count-weighted mean.",{"id":200,"topic":9,"difficulty":161,"body":201,"options":202,"correct_key":139,"explanation":211},"01a05d13-8d2e-7d9b-9358-5c3b1aebface","In DDP, gradient 'bucketing' groups multiple parameters' gradients into one buffer before running a single all-reduce over that buffer. What is the main reason DDP does this instead of running one all-reduce per parameter?",[203,205,207,209],{"key":133,"text":204},"It reduces per-rank GPU and host memory footprint, since bucketed gradients replace the need to keep separate per-parameter communication buffers alive.",{"key":136,"text":206},"It's mathematically required; naive per-parameter averaging is not numerically valid without first grouping tensors.",{"key":139,"text":208},"It cuts per-call overhead and overlaps with backward.",{"key":142,"text":210},"It is intended to make results bit-identical across backends.","Launching a separate collective per parameter would mean hundreds of tiny, high-overhead calls. Bucketing batches gradients into fewer, larger all-reduce calls, and — because backward computes gradients layer by layer — a bucket that becomes fully ready can start communicating while backward keeps computing gradients for other layers, overlapping communication with compute.",{"fields":213,"seniorities":436,"interview_shapes":437,"locales":442,"oauth":444,"question_count":447,"coach_enabled":448,"jd_match_enabled":448},[214,239,259,275,299,312,331,350,372,391,406,428],{"key":215,"name_tr":216,"name_en":216,"sort":147,"specializations":217},"backend","Backend",[218,221,224,227,230,233,236],{"key":219,"name":220,"field":215},"general","Genel",{"key":222,"name":223,"field":215},"go","Go",{"key":225,"name":226,"field":215},"python","Python",{"key":228,"name":229,"field":215},"java","Java",{"key":231,"name":232,"field":215},"csharp","C#\u002F.NET",{"key":234,"name":235,"field":215},"nodejs","Node.js",{"key":237,"name":238,"field":215},"php","PHP",{"key":240,"name_tr":241,"name_en":241,"sort":161,"specializations":242},"frontend","Frontend",[243,244,247,250,253,256],{"key":219,"name":220,"field":240},{"key":245,"name":246,"field":240},"javascript","JavaScript",{"key":248,"name":249,"field":240},"typescript","TypeScript",{"key":251,"name":252,"field":240},"react","React",{"key":254,"name":255,"field":240},"vue","Vue",{"key":257,"name":258,"field":240},"angular","Angular",{"key":260,"name_tr":261,"name_en":261,"sort":129,"specializations":262},"fullstack","Fullstack",[263,264,265,266,267,268,269,270,271,272,273,274],{"key":219,"name":220,"field":260},{"key":222,"name":223,"field":215},{"key":225,"name":226,"field":215},{"key":228,"name":229,"field":215},{"key":231,"name":232,"field":215},{"key":234,"name":235,"field":215},{"key":237,"name":238,"field":215},{"key":245,"name":246,"field":240},{"key":248,"name":249,"field":240},{"key":251,"name":252,"field":240},{"key":254,"name":255,"field":240},{"key":257,"name":258,"field":240},{"key":276,"name_tr":277,"name_en":277,"sort":278,"specializations":279},"devops-cloud","DevOps \u002F Cloud",4,[280,281,284,287,290,293,296],{"key":219,"name":220,"field":276},{"key":282,"name":283,"field":276},"aws","AWS",{"key":285,"name":286,"field":276},"gcp","GCP",{"key":288,"name":289,"field":276},"azure","Azure",{"key":291,"name":292,"field":276},"kubernetes","Kubernetes",{"key":294,"name":295,"field":276},"terraform","Terraform",{"key":297,"name":298,"field":276},"linux","Linux",{"key":300,"name_tr":301,"name_en":301,"sort":302,"specializations":303},"ai-engineer","AI Engineer",5,[304,305,306,309],{"key":219,"name":220,"field":300},{"key":225,"name":226,"field":300},{"key":307,"name":308,"field":300},"llm-rag","LLM\u002FRAG",{"key":310,"name":311,"field":300},"mlops","MLOps",{"key":313,"name_tr":314,"name_en":315,"sort":316,"specializations":317},"database","Veritabanı","Database",6,[318,319,322,325,328],{"key":219,"name":220,"field":313},{"key":320,"name":321,"field":313},"postgresql","PostgreSQL",{"key":323,"name":324,"field":313},"mysql","MySQL",{"key":326,"name":327,"field":313},"mongodb","MongoDB",{"key":329,"name":330,"field":313},"redis","Redis",{"key":332,"name_tr":333,"name_en":334,"sort":335,"specializations":336},"mobile","Mobil","Mobile",7,[337,338,341,344,347],{"key":219,"name":220,"field":332},{"key":339,"name":340,"field":332},"ios-swift","iOS (Swift)",{"key":342,"name":343,"field":332},"android-kotlin","Android (Kotlin)",{"key":345,"name":346,"field":332},"flutter","Flutter",{"key":348,"name":349,"field":332},"react-native","React Native",{"key":351,"name_tr":352,"name_en":353,"sort":354,"specializations":355},"security","Güvenlik","Security",8,[356,357,360,363,366,369],{"key":219,"name":220,"field":351},{"key":358,"name":359,"field":351},"appsec","AppSec",{"key":361,"name":362,"field":351},"offensive-pentest","Offensive \u002F Pentest",{"key":364,"name":365,"field":351},"cloud-security","Cloud Security",{"key":367,"name":368,"field":351},"devsecops","DevSecOps",{"key":370,"name":371,"field":351},"blue-team-incident","Blue Team \u002F Incident",{"key":373,"name_tr":374,"name_en":375,"sort":376,"specializations":377},"qa-test-automation","QA \u002F Test Otomasyonu","QA \u002F Test Automation",9,[378,379,382,385,388],{"key":219,"name":220,"field":373},{"key":380,"name":381,"field":373},"test-automation","Test Automation",{"key":383,"name":384,"field":373},"sdet","SDET",{"key":386,"name":387,"field":373},"performance-testing","Performance Testing",{"key":389,"name":390,"field":373},"mobile-qa","Mobile QA",{"key":392,"name_tr":393,"name_en":393,"sort":394,"specializations":395},"data-engineer","Data Engineer",10,[396,397,400,403],{"key":219,"name":220,"field":392},{"key":398,"name":399,"field":392},"pipelines-etl","Pipelines \u002F ETL",{"key":401,"name":402,"field":392},"streaming","Streaming",{"key":404,"name":405,"field":392},"warehousing","Warehousing",{"key":407,"name_tr":408,"name_en":409,"sort":410,"specializations":411},"game-dev","Oyun Geliştirme","Game Development",11,[412,413,416,419,422,425],{"key":219,"name":220,"field":407},{"key":414,"name":415,"field":407},"unity-csharp","Unity (C#)",{"key":417,"name":418,"field":407},"unreal-cpp","Unreal (C++)",{"key":420,"name":421,"field":407},"gameplay","Gameplay",{"key":423,"name":424,"field":407},"graphics-rendering","Graphics \u002F Rendering",{"key":426,"name":427,"field":407},"multiplayer-netcode","Multiplayer \u002F Netcode",{"key":5,"name_tr":6,"name_en":6,"sort":429,"specializations":430},12,[431,432,433,434,435],{"key":219,"name":220,"field":5},{"key":114,"name":115,"field":5},{"key":121,"name":122,"field":5},{"key":118,"name":119,"field":5},{"key":124,"name":125,"field":5},[14,15,16],{"junior":438,"mid":440,"senior":441},{"questions":439,"median_sec":3},20,{"questions":439,"median_sec":3},{"questions":439,"median_sec":3},[443,10],"tr",[445,446],"google","github",28950,true]