[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"me":3,"catalog:en:data-engineer\u002Fpe-spark-api-semantics-udfs":4,"config":190},null,{"field_key":5,"field_name":6,"seniority":7,"topic_key":8,"topic_name":9,"spec_key":7,"spec_name":7,"locale":10,"cell_total":11,"field_total":12,"seniorities":13,"topics":17,"specs":94,"samples":105},"data-engineer","Data Engineer","","pe-spark-api-semantics-udfs","Pe Spark Api Semantics Udfs","en",75,1950,[14,15,16],"junior","mid","senior",[18,21,24,27,30,33,36,39,42,45,48,51,52,55,58,61,64,67,70,73,76,79,82,85,88,91],{"key":19,"name":20,"count":11},"data-governance-lineage","Data Governance Lineage",{"key":22,"name":23,"count":11},"data-modeling-warehousing","Data Modeling Warehousing",{"key":25,"name":26,"count":11},"data-partitioning-scaling","Data Partitioning Scaling",{"key":28,"name":29,"count":11},"data-pipeline-design","Data Pipeline Design",{"key":31,"name":32,"count":11},"data-pipeline-orchestration","Data Pipeline Orchestration",{"key":34,"name":35,"count":11},"data-pipeline-reliability","Data Pipeline Reliability",{"key":37,"name":38,"count":11},"data-quality-validation","Data Quality Validation",{"key":40,"name":41,"count":11},"pe-airflow-scheduling-dependencies","Pe Airflow Scheduling Dependencies",{"key":43,"name":44,"count":11},"pe-dbt-models-materializations-tests","Pe Dbt Models Materializations Tests",{"key":46,"name":47,"count":11},"pe-file-formats-table-formats","Pe File Formats Table Formats",{"key":49,"name":50,"count":11},"pe-incremental-cdc-merge-mechanics","Pe Incremental Cdc Merge Mechanics",{"key":8,"name":9,"count":11},{"key":53,"name":54,"count":11},"pe-spark-execution-shuffle-partitions","Pe Spark Execution Shuffle Partitions",{"key":56,"name":57,"count":11},"st-flink-runtime-checkpointing-state","St Flink Runtime Checkpointing State",{"key":59,"name":60,"count":11},"st-flink-time-watermarks-windows","St Flink Time Watermarks Windows",{"key":62,"name":63,"count":11},"st-kafka-broker-storage-replication","St Kafka Broker Storage Replication",{"key":65,"name":66,"count":11},"st-kafka-connect-schema-registry","St Kafka Connect Schema Registry",{"key":68,"name":69,"count":11},"st-kafka-producer-consumer-semantics","St Kafka Producer Consumer Semantics",{"key":71,"name":72,"count":11},"streaming-fundamentals","Streaming Fundamentals",{"key":74,"name":75,"count":11},"st-streaming-joins-tables","St Streaming Joins Tables",{"key":77,"name":78,"count":11},"wh-bigquery-execution-slots-cost","Wh Bigquery Execution Slots Cost",{"key":80,"name":81,"count":11},"wh-bigquery-storage-partitioning-clustering","Wh Bigquery Storage Partitioning Clustering",{"key":83,"name":84,"count":11},"wh-materialized-views-caching","Wh Materialized Views Caching",{"key":86,"name":87,"count":11},"wh-redshift-distribution-sort-vacuum","Wh Redshift Distribution Sort Vacuum",{"key":89,"name":90,"count":11},"wh-snowflake-compute-warehouses","Wh Snowflake Compute Warehouses",{"key":92,"name":93,"count":11},"wh-snowflake-storage-micropartitions","Wh Snowflake Storage Micropartitions",[95,99,102],{"key":96,"name":97,"count":98},"pipelines-etl","Pipelines \u002F ETL",450,{"key":100,"name":101,"count":98},"streaming","Streaming",{"key":103,"name":104,"count":98},"warehousing","Warehousing",[106,124,138,151,164,177],{"id":107,"topic":9,"difficulty":108,"body":109,"options":110,"correct_key":115,"explanation":123},"01a00ae7-b517-744b-a4ab-e069ed0e13ac",2,"In Spark's Catalyst optimizer, a DataFrame query goes through several plan stages before execution. What is the correct order?",[111,114,117,120],{"key":112,"text":113},"a","Physical plan → optimized logical plan → logical plan",{"key":115,"text":116},"b","Logical plan → optimized logical plan → physical plan",{"key":118,"text":119},"c","Optimized logical plan → logical plan → physical plan",{"key":121,"text":122},"d","Physical plan → logical plan → optimized logical plan","Catalyst first builds a logical plan from the DataFrame\u002FSQL code, applies rule-based optimizations to produce the optimized logical plan, and finally generates the physical plan that the execution engine runs.",{"id":125,"topic":9,"difficulty":126,"body":127,"options":128,"correct_key":112,"explanation":137},"01a00ae7-b518-7626-abe4-2912eea0e00d",1,"```python\ndf.filter(df.amount > 100).explain()\n```\nCalling `.explain()` with no arguments on a DataFrame prints which plan by default?",[129,131,133,135],{"key":112,"text":130},"Only the physical plan",{"key":115,"text":132},"Only the unresolved logical plan",{"key":118,"text":134},"All four plan stages (parsed, analyzed, optimized, physical)",{"key":121,"text":136},"Only the optimized logical plan","`explain()` with no arguments (extended=False) prints just the physical plan. Passing extended=True or a mode string like 'formatted' shows more detail, including the logical plan stages.",{"id":139,"topic":9,"difficulty":108,"body":140,"options":141,"correct_key":121,"explanation":150},"01a00ae7-b519-7ba9-bf03-b488d145c498","In Catalyst's optimized logical plan, what does 'column pruning' do?",[142,144,146,148],{"key":112,"text":143},"Removes rows that fail a filter condition before scan",{"key":115,"text":145},"Merges two scans of the same table into one",{"key":118,"text":147},"Reorders join operations by estimated cost",{"key":121,"text":149},"Drops columns never referenced downstream","Column pruning finds which columns are actually used later in the plan and rewrites the scan to read only those columns, reducing I\u002FO especially for columnar formats.",{"id":152,"topic":9,"difficulty":108,"body":153,"options":154,"correct_key":118,"explanation":163},"01a00ae7-b51c-7676-a838-ea98052a861d","What does Catalyst's constant folding optimization do with an expression like `WHERE price > 10 + 5`?",[155,157,159,161],{"key":112,"text":156},"It pushes the whole expression down to the storage layer unchanged",{"key":115,"text":158},"It rewrites the comparison into a join condition",{"key":118,"text":160},"Computes `10 + 5` at plan time and substitutes `15`",{"key":121,"text":162},"It converts the filter into a window function","Constant folding evaluates expressions made only of literals during query planning, so the physical plan compares `price > 15` directly instead of recomputing `10 + 5` for every row.",{"id":165,"topic":9,"difficulty":108,"body":166,"options":167,"correct_key":115,"explanation":176},"01a00ae7-b51f-7546-9843-17734e1bc90a","What does 'predicate pushdown' mean when Spark reads from a filterable data source?",[168,170,172,174],{"key":112,"text":169},"Filters are moved to run after the join stage instead of before",{"key":115,"text":171},"The filter is passed to the source to skip non-matching data early",{"key":118,"text":173},"Filters are converted into UDFs for flexibility",{"key":121,"text":175},"All filters are evaluated only on the driver","With predicate pushdown, Spark translates a filter condition into something the source (e.g. JDBC, Parquet) can apply itself, so fewer rows or files are actually read into the cluster.",{"id":178,"topic":9,"difficulty":108,"body":179,"options":180,"correct_key":121,"explanation":189},"01a00ae7-b521-77f8-91fb-99e733a4d18a","A team writes:\n```python\ndf.filter(is_valid_udf(df.email))\n```\nwhere `is_valid_udf` is a Python UDF, then reads `explain()`. What should they expect regarding pushdown to the underlying Parquet source?",[181,183,185,187],{"key":112,"text":182},"The UDF filter is automatically pushed down because Parquet supports arbitrary filters",{"key":115,"text":184},"Catalyst rewrites the UDF into a built-in expression before pushdown",{"key":118,"text":186},"Pushdown still happens because Catalyst inlines the UDF bytecode",{"key":121,"text":188},"The UDF filter cannot be pushed down; Catalyst evaluates it after the scan","Catalyst can only push down filters it understands as expressions. A UDF is opaque to the optimizer, so the filter it's part of is evaluated inside Spark after rows are read, not pushed into the Parquet reader.",{"fields":191,"seniorities":408,"interview_shapes":409,"locales":414,"oauth":416,"question_count":419,"coach_enabled":420,"jd_match_enabled":420},[192,217,237,254,278,291,310,329,351,370,377,399],{"key":193,"name_tr":194,"name_en":194,"sort":126,"specializations":195},"backend","Backend",[196,199,202,205,208,211,214],{"key":197,"name":198,"field":193},"general","Genel",{"key":200,"name":201,"field":193},"go","Go",{"key":203,"name":204,"field":193},"python","Python",{"key":206,"name":207,"field":193},"java","Java",{"key":209,"name":210,"field":193},"csharp","C#\u002F.NET",{"key":212,"name":213,"field":193},"nodejs","Node.js",{"key":215,"name":216,"field":193},"php","PHP",{"key":218,"name_tr":219,"name_en":219,"sort":108,"specializations":220},"frontend","Frontend",[221,222,225,228,231,234],{"key":197,"name":198,"field":218},{"key":223,"name":224,"field":218},"javascript","JavaScript",{"key":226,"name":227,"field":218},"typescript","TypeScript",{"key":229,"name":230,"field":218},"react","React",{"key":232,"name":233,"field":218},"vue","Vue",{"key":235,"name":236,"field":218},"angular","Angular",{"key":238,"name_tr":239,"name_en":239,"sort":240,"specializations":241},"fullstack","Fullstack",3,[242,243,244,245,246,247,248,249,250,251,252,253],{"key":197,"name":198,"field":238},{"key":200,"name":201,"field":193},{"key":203,"name":204,"field":193},{"key":206,"name":207,"field":193},{"key":209,"name":210,"field":193},{"key":212,"name":213,"field":193},{"key":215,"name":216,"field":193},{"key":223,"name":224,"field":218},{"key":226,"name":227,"field":218},{"key":229,"name":230,"field":218},{"key":232,"name":233,"field":218},{"key":235,"name":236,"field":218},{"key":255,"name_tr":256,"name_en":256,"sort":257,"specializations":258},"devops-cloud","DevOps \u002F Cloud",4,[259,260,263,266,269,272,275],{"key":197,"name":198,"field":255},{"key":261,"name":262,"field":255},"aws","AWS",{"key":264,"name":265,"field":255},"gcp","GCP",{"key":267,"name":268,"field":255},"azure","Azure",{"key":270,"name":271,"field":255},"kubernetes","Kubernetes",{"key":273,"name":274,"field":255},"terraform","Terraform",{"key":276,"name":277,"field":255},"linux","Linux",{"key":279,"name_tr":280,"name_en":280,"sort":281,"specializations":282},"ai-engineer","AI Engineer",5,[283,284,285,288],{"key":197,"name":198,"field":279},{"key":203,"name":204,"field":279},{"key":286,"name":287,"field":279},"llm-rag","LLM\u002FRAG",{"key":289,"name":290,"field":279},"mlops","MLOps",{"key":292,"name_tr":293,"name_en":294,"sort":295,"specializations":296},"database","Veritabanı","Database",6,[297,298,301,304,307],{"key":197,"name":198,"field":292},{"key":299,"name":300,"field":292},"postgresql","PostgreSQL",{"key":302,"name":303,"field":292},"mysql","MySQL",{"key":305,"name":306,"field":292},"mongodb","MongoDB",{"key":308,"name":309,"field":292},"redis","Redis",{"key":311,"name_tr":312,"name_en":313,"sort":314,"specializations":315},"mobile","Mobil","Mobile",7,[316,317,320,323,326],{"key":197,"name":198,"field":311},{"key":318,"name":319,"field":311},"ios-swift","iOS (Swift)",{"key":321,"name":322,"field":311},"android-kotlin","Android (Kotlin)",{"key":324,"name":325,"field":311},"flutter","Flutter",{"key":327,"name":328,"field":311},"react-native","React Native",{"key":330,"name_tr":331,"name_en":332,"sort":333,"specializations":334},"security","Güvenlik","Security",8,[335,336,339,342,345,348],{"key":197,"name":198,"field":330},{"key":337,"name":338,"field":330},"appsec","AppSec",{"key":340,"name":341,"field":330},"offensive-pentest","Offensive \u002F Pentest",{"key":343,"name":344,"field":330},"cloud-security","Cloud Security",{"key":346,"name":347,"field":330},"devsecops","DevSecOps",{"key":349,"name":350,"field":330},"blue-team-incident","Blue Team \u002F Incident",{"key":352,"name_tr":353,"name_en":354,"sort":355,"specializations":356},"qa-test-automation","QA \u002F Test Otomasyonu","QA \u002F Test Automation",9,[357,358,361,364,367],{"key":197,"name":198,"field":352},{"key":359,"name":360,"field":352},"test-automation","Test Automation",{"key":362,"name":363,"field":352},"sdet","SDET",{"key":365,"name":366,"field":352},"performance-testing","Performance Testing",{"key":368,"name":369,"field":352},"mobile-qa","Mobile QA",{"key":5,"name_tr":6,"name_en":6,"sort":371,"specializations":372},10,[373,374,375,376],{"key":197,"name":198,"field":5},{"key":96,"name":97,"field":5},{"key":100,"name":101,"field":5},{"key":103,"name":104,"field":5},{"key":378,"name_tr":379,"name_en":380,"sort":381,"specializations":382},"game-dev","Oyun Geliştirme","Game Development",11,[383,384,387,390,393,396],{"key":197,"name":198,"field":378},{"key":385,"name":386,"field":378},"unity-csharp","Unity (C#)",{"key":388,"name":389,"field":378},"unreal-cpp","Unreal (C++)",{"key":391,"name":392,"field":378},"gameplay","Gameplay",{"key":394,"name":395,"field":378},"graphics-rendering","Graphics \u002F Rendering",{"key":397,"name":398,"field":378},"multiplayer-netcode","Multiplayer \u002F Netcode",{"key":400,"name_tr":401,"name_en":401,"sort":402,"specializations":403},"ml-engineer","ML Engineer",12,[404,405],{"key":197,"name":198,"field":400},{"key":406,"name":407,"field":400},"classical-ml","Classical ML",[14,15,16],{"junior":410,"mid":412,"senior":413},{"questions":411,"median_sec":3},20,{"questions":411,"median_sec":3},{"questions":411,"median_sec":3},[415,10],"tr",[417,418],"google","github",27600,true]