[{"data":1,"prerenderedAt":1100},["ShallowReactive",2],{"content-query-YR11CxcxC0":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"body":10,"_type":1094,"_id":1095,"_source":1096,"_file":1097,"_stem":1098,"_extension":1099},"\u002Fdocs\u002Ffeatures\u002Forchestration","features",false,"","Orchestration","Schedule and manage data pipelines and workflows",{"type":11,"children":12,"toc":1046},"root",[13,21,27,34,41,46,76,82,117,123,129,134,157,163,173,213,223,248,258,289,299,349,355,361,366,374,380,385,428,434,440,445,473,479,484,492,498,521,527,533,538,546,552,557,613,619,625,630,638,644,649,657,663,669,677,683,691,697,705,711,717,722,745,751,756,764,770,775,798,804,810,833,839,862,868,891,897,903,939,945,968,974,997,1003,1040],{"type":14,"tag":15,"props":16,"children":18},"element","h1",{"id":17},"orchestration",[19],{"type":20,"value":8},"text",{"type":14,"tag":22,"props":23,"children":24},"p",{},[25],{"type":20,"value":26},"Orchestrate complex data workflows. Schedule jobs, manage dependencies, and automate data processing pipelines.",{"type":14,"tag":28,"props":29,"children":31},"h2",{"id":30},"getting-started",[32],{"type":20,"value":33},"Getting Started",{"type":14,"tag":35,"props":36,"children":38},"h3",{"id":37},"what-is-orchestration",[39],{"type":20,"value":40},"What is Orchestration?",{"type":14,"tag":22,"props":42,"children":43},{},[44],{"type":20,"value":45},"Orchestration lets you:",{"type":14,"tag":47,"props":48,"children":49},"ul",{},[50,56,61,66,71],{"type":14,"tag":51,"props":52,"children":53},"li",{},[54],{"type":20,"value":55},"Define data pipelines with multiple steps",{"type":14,"tag":51,"props":57,"children":58},{},[59],{"type":20,"value":60},"Schedule automatic execution (hourly, daily, weekly)",{"type":14,"tag":51,"props":62,"children":63},{},[64],{"type":20,"value":65},"Handle dependencies between steps",{"type":14,"tag":51,"props":67,"children":68},{},[69],{"type":20,"value":70},"Monitor job execution",{"type":14,"tag":51,"props":72,"children":73},{},[74],{"type":20,"value":75},"Retry on failures",{"type":14,"tag":35,"props":77,"children":79},{"id":78},"creating-your-first-pipeline",[80],{"type":20,"value":81},"Creating Your First Pipeline",{"type":14,"tag":83,"props":84,"children":85},"ol",{},[86,98,107,112],{"type":14,"tag":51,"props":87,"children":88},{},[89,91,96],{"type":20,"value":90},"Click ",{"type":14,"tag":92,"props":93,"children":94},"strong",{},[95],{"type":20,"value":8},{"type":20,"value":97}," in sidebar",{"type":14,"tag":51,"props":99,"children":100},{},[101,102],{"type":20,"value":90},{"type":14,"tag":92,"props":103,"children":104},{},[105],{"type":20,"value":106},"New Pipeline",{"type":14,"tag":51,"props":108,"children":109},{},[110],{"type":20,"value":111},"Define workflow steps",{"type":14,"tag":51,"props":113,"children":114},{},[115],{"type":20,"value":116},"Set schedule and triggers",{"type":14,"tag":28,"props":118,"children":120},{"id":119},"defining-workflows",[121],{"type":20,"value":122},"Defining Workflows",{"type":14,"tag":35,"props":124,"children":126},{"id":125},"pipeline-steps",[127],{"type":20,"value":128},"Pipeline Steps",{"type":14,"tag":22,"props":130,"children":131},{},[132],{"type":20,"value":133},"Each step is a task that:",{"type":14,"tag":47,"props":135,"children":136},{},[137,142,147,152],{"type":14,"tag":51,"props":138,"children":139},{},[140],{"type":20,"value":141},"Queries your database",{"type":14,"tag":51,"props":143,"children":144},{},[145],{"type":20,"value":146},"Processes data",{"type":14,"tag":51,"props":148,"children":149},{},[150],{"type":20,"value":151},"Runs a function",{"type":14,"tag":51,"props":153,"children":154},{},[155],{"type":20,"value":156},"Triggers another step",{"type":14,"tag":35,"props":158,"children":160},{"id":159},"step-types",[161],{"type":20,"value":162},"Step Types",{"type":14,"tag":22,"props":164,"children":165},{},[166,171],{"type":14,"tag":92,"props":167,"children":168},{},[169],{"type":20,"value":170},"Query Steps",{"type":20,"value":172}," - Run SQL or code:",{"type":14,"tag":174,"props":175,"children":179},"pre",{"code":176,"language":177,"meta":7,"className":178,"style":7},"-- Extract users from database\nSELECT user_id, created_at FROM users \nWHERE created_at > CURRENT_DATE - INTERVAL '7 days'\n","sql","language-sql shiki shiki-themes github-dark",[180],{"type":14,"tag":181,"props":182,"children":183},"code",{"__ignoreMap":7},[184,195,204],{"type":14,"tag":185,"props":186,"children":189},"span",{"class":187,"line":188},"line",1,[190],{"type":14,"tag":185,"props":191,"children":192},{},[193],{"type":20,"value":194},"-- Extract users from database\n",{"type":14,"tag":185,"props":196,"children":198},{"class":187,"line":197},2,[199],{"type":14,"tag":185,"props":200,"children":201},{},[202],{"type":20,"value":203},"SELECT user_id, created_at FROM users \n",{"type":14,"tag":185,"props":205,"children":207},{"class":187,"line":206},3,[208],{"type":14,"tag":185,"props":209,"children":210},{},[211],{"type":20,"value":212},"WHERE created_at > CURRENT_DATE - INTERVAL '7 days'\n",{"type":14,"tag":22,"props":214,"children":215},{},[216,221],{"type":14,"tag":92,"props":217,"children":218},{},[219],{"type":20,"value":220},"Transform Steps",{"type":20,"value":222}," - Process data:",{"type":14,"tag":174,"props":224,"children":228},{"code":225,"language":226,"meta":7,"className":227,"style":7},"# Aggregate data\nusers_per_day = data.groupby('created_at').size()\n","python","language-python shiki shiki-themes github-dark",[229],{"type":14,"tag":181,"props":230,"children":231},{"__ignoreMap":7},[232,240],{"type":14,"tag":185,"props":233,"children":234},{"class":187,"line":188},[235],{"type":14,"tag":185,"props":236,"children":237},{},[238],{"type":20,"value":239},"# Aggregate data\n",{"type":14,"tag":185,"props":241,"children":242},{"class":187,"line":197},[243],{"type":14,"tag":185,"props":244,"children":245},{},[246],{"type":20,"value":247},"users_per_day = data.groupby('created_at').size()\n",{"type":14,"tag":22,"props":249,"children":250},{},[251,256],{"type":14,"tag":92,"props":252,"children":253},{},[254],{"type":20,"value":255},"Function Steps",{"type":20,"value":257}," - Call your code:",{"type":14,"tag":174,"props":259,"children":261},{"code":260,"language":226,"meta":7,"className":227,"style":7},"def send_notifications(users):\n    for user in users:\n        send_email(user.email, \"Weekly Report\")\n",[262],{"type":14,"tag":181,"props":263,"children":264},{"__ignoreMap":7},[265,273,281],{"type":14,"tag":185,"props":266,"children":267},{"class":187,"line":188},[268],{"type":14,"tag":185,"props":269,"children":270},{},[271],{"type":20,"value":272},"def send_notifications(users):\n",{"type":14,"tag":185,"props":274,"children":275},{"class":187,"line":197},[276],{"type":14,"tag":185,"props":277,"children":278},{},[279],{"type":20,"value":280},"    for user in users:\n",{"type":14,"tag":185,"props":282,"children":283},{"class":187,"line":206},[284],{"type":14,"tag":185,"props":285,"children":286},{},[287],{"type":20,"value":288},"        send_email(user.email, \"Weekly Report\")\n",{"type":14,"tag":22,"props":290,"children":291},{},[292,297],{"type":14,"tag":92,"props":293,"children":294},{},[295],{"type":20,"value":296},"Export Steps",{"type":20,"value":298}," - Output results:",{"type":14,"tag":174,"props":300,"children":302},{"code":301,"language":226,"meta":7,"className":227,"style":7},"# Save to database\nresults.to_credvault(collection='reports')\n\n# Send to external system\npush_to_slack(channel='#data-team', message=summary)\n",[303],{"type":14,"tag":181,"props":304,"children":305},{"__ignoreMap":7},[306,314,322,331,340],{"type":14,"tag":185,"props":307,"children":308},{"class":187,"line":188},[309],{"type":14,"tag":185,"props":310,"children":311},{},[312],{"type":20,"value":313},"# Save to database\n",{"type":14,"tag":185,"props":315,"children":316},{"class":187,"line":197},[317],{"type":14,"tag":185,"props":318,"children":319},{},[320],{"type":20,"value":321},"results.to_credvault(collection='reports')\n",{"type":14,"tag":185,"props":323,"children":324},{"class":187,"line":206},[325],{"type":14,"tag":185,"props":326,"children":328},{"emptyLinePlaceholder":327},true,[329],{"type":20,"value":330},"\n",{"type":14,"tag":185,"props":332,"children":334},{"class":187,"line":333},4,[335],{"type":14,"tag":185,"props":336,"children":337},{},[338],{"type":20,"value":339},"# Send to external system\n",{"type":14,"tag":185,"props":341,"children":343},{"class":187,"line":342},5,[344],{"type":14,"tag":185,"props":345,"children":346},{},[347],{"type":20,"value":348},"push_to_slack(channel='#data-team', message=summary)\n",{"type":14,"tag":28,"props":350,"children":352},{"id":351},"scheduling-pipelines",[353],{"type":20,"value":354},"Scheduling Pipelines",{"type":14,"tag":35,"props":356,"children":358},{"id":357},"cron-expressions",[359],{"type":20,"value":360},"Cron Expressions",{"type":14,"tag":22,"props":362,"children":363},{},[364],{"type":20,"value":365},"Define when pipelines run:",{"type":14,"tag":174,"props":367,"children":369},{"code":368},"0 9 * * * ........... Every day at 9 AM\n0 *\u002F6 * * * ......... Every 6 hours\n0 0 * * MON ......... Every Monday at midnight\n0 0 1 * * ........... First day of every month\n",[370],{"type":14,"tag":181,"props":371,"children":372},{"__ignoreMap":7},[373],{"type":20,"value":368},{"type":14,"tag":35,"props":375,"children":377},{"id":376},"triggers",[378],{"type":20,"value":379},"Triggers",{"type":14,"tag":22,"props":381,"children":382},{},[383],{"type":20,"value":384},"Pipelines run based on:",{"type":14,"tag":47,"props":386,"children":387},{},[388,398,408,418],{"type":14,"tag":51,"props":389,"children":390},{},[391,396],{"type":14,"tag":92,"props":392,"children":393},{},[394],{"type":20,"value":395},"Schedule",{"type":20,"value":397}," - Cron expression",{"type":14,"tag":51,"props":399,"children":400},{},[401,406],{"type":14,"tag":92,"props":402,"children":403},{},[404],{"type":20,"value":405},"Event",{"type":20,"value":407}," - Database change, webhook, external event",{"type":14,"tag":51,"props":409,"children":410},{},[411,416],{"type":14,"tag":92,"props":412,"children":413},{},[414],{"type":20,"value":415},"Manual",{"type":20,"value":417}," - User clicks Run",{"type":14,"tag":51,"props":419,"children":420},{},[421,426],{"type":14,"tag":92,"props":422,"children":423},{},[424],{"type":20,"value":425},"On-demand",{"type":20,"value":427}," - Via API call",{"type":14,"tag":28,"props":429,"children":431},{"id":430},"monitoring-pipelines",[432],{"type":20,"value":433},"Monitoring Pipelines",{"type":14,"tag":35,"props":435,"children":437},{"id":436},"view-execution-history",[438],{"type":20,"value":439},"View Execution History",{"type":14,"tag":22,"props":441,"children":442},{},[443],{"type":20,"value":444},"See all pipeline runs:",{"type":14,"tag":47,"props":446,"children":447},{},[448,453,458,463,468],{"type":14,"tag":51,"props":449,"children":450},{},[451],{"type":20,"value":452},"Start and end time",{"type":14,"tag":51,"props":454,"children":455},{},[456],{"type":20,"value":457},"Status (success, failed, running)",{"type":14,"tag":51,"props":459,"children":460},{},[461],{"type":20,"value":462},"Duration",{"type":14,"tag":51,"props":464,"children":465},{},[466],{"type":20,"value":467},"Output results",{"type":14,"tag":51,"props":469,"children":470},{},[471],{"type":20,"value":472},"Error messages",{"type":14,"tag":35,"props":474,"children":476},{"id":475},"pipeline-logs",[477],{"type":20,"value":478},"Pipeline Logs",{"type":14,"tag":22,"props":480,"children":481},{},[482],{"type":20,"value":483},"Click any run to see detailed logs:",{"type":14,"tag":174,"props":485,"children":487},{"code":486},"2024-06-12 09:00:00 - Pipeline started\n2024-06-12 09:00:01 - Step 1: Extract data (127 records)\n2024-06-12 09:00:05 - Step 2: Transform data (127 → 98 after filtering)\n2024-06-12 09:00:08 - Step 3: Load to database (98 inserted)\n2024-06-12 09:00:09 - Pipeline completed successfully\n",[488],{"type":14,"tag":181,"props":489,"children":490},{"__ignoreMap":7},[491],{"type":20,"value":486},{"type":14,"tag":35,"props":493,"children":495},{"id":494},"debugging-failed-runs",[496],{"type":20,"value":497},"Debugging Failed Runs",{"type":14,"tag":83,"props":499,"children":500},{},[501,506,511,516],{"type":14,"tag":51,"props":502,"children":503},{},[504],{"type":20,"value":505},"Click failed run",{"type":14,"tag":51,"props":507,"children":508},{},[509],{"type":20,"value":510},"See error message",{"type":14,"tag":51,"props":512,"children":513},{},[514],{"type":20,"value":515},"Check logs",{"type":14,"tag":51,"props":517,"children":518},{},[519],{"type":20,"value":520},"Fix and rerun",{"type":14,"tag":28,"props":522,"children":524},{"id":523},"error-handling",[525],{"type":20,"value":526},"Error Handling",{"type":14,"tag":35,"props":528,"children":530},{"id":529},"retry-policies",[531],{"type":20,"value":532},"Retry Policies",{"type":14,"tag":22,"props":534,"children":535},{},[536],{"type":20,"value":537},"Configure what happens on failure:",{"type":14,"tag":174,"props":539,"children":541},{"code":540},"Max retries: 3\nWait between retries: 60 seconds\nOn final failure: Send alert\n",[542],{"type":14,"tag":181,"props":543,"children":544},{"__ignoreMap":7},[545],{"type":20,"value":540},{"type":14,"tag":35,"props":547,"children":549},{"id":548},"conditional-steps",[550],{"type":20,"value":551},"Conditional Steps",{"type":14,"tag":22,"props":553,"children":554},{},[555],{"type":20,"value":556},"Skip or branch based on results:",{"type":14,"tag":174,"props":558,"children":560},{"code":559,"language":226,"meta":7,"className":227,"style":7},"if records_count > 0:\n    # Process records\n    send_notification(f\"Processed {records_count}\")\nelse:\n    # No data, skip processing\n    log_message(\"No new data\")\n",[561],{"type":14,"tag":181,"props":562,"children":563},{"__ignoreMap":7},[564,572,580,588,596,604],{"type":14,"tag":185,"props":565,"children":566},{"class":187,"line":188},[567],{"type":14,"tag":185,"props":568,"children":569},{},[570],{"type":20,"value":571},"if records_count > 0:\n",{"type":14,"tag":185,"props":573,"children":574},{"class":187,"line":197},[575],{"type":14,"tag":185,"props":576,"children":577},{},[578],{"type":20,"value":579},"    # Process records\n",{"type":14,"tag":185,"props":581,"children":582},{"class":187,"line":206},[583],{"type":14,"tag":185,"props":584,"children":585},{},[586],{"type":20,"value":587},"    send_notification(f\"Processed {records_count}\")\n",{"type":14,"tag":185,"props":589,"children":590},{"class":187,"line":333},[591],{"type":14,"tag":185,"props":592,"children":593},{},[594],{"type":20,"value":595},"else:\n",{"type":14,"tag":185,"props":597,"children":598},{"class":187,"line":342},[599],{"type":14,"tag":185,"props":600,"children":601},{},[602],{"type":20,"value":603},"    # No data, skip processing\n",{"type":14,"tag":185,"props":605,"children":607},{"class":187,"line":606},6,[608],{"type":14,"tag":185,"props":609,"children":610},{},[611],{"type":20,"value":612},"    log_message(\"No new data\")\n",{"type":14,"tag":28,"props":614,"children":616},{"id":615},"dependencies-and-ordering",[617],{"type":20,"value":618},"Dependencies and Ordering",{"type":14,"tag":35,"props":620,"children":622},{"id":621},"sequential-steps",[623],{"type":20,"value":624},"Sequential Steps",{"type":14,"tag":22,"props":626,"children":627},{},[628],{"type":20,"value":629},"Steps run one after another:",{"type":14,"tag":174,"props":631,"children":633},{"code":632},"1. Extract data from database\n   ↓\n2. Validate data quality\n   ↓\n3. Transform and aggregate\n   ↓\n4. Load to destination\n",[634],{"type":14,"tag":181,"props":635,"children":636},{"__ignoreMap":7},[637],{"type":20,"value":632},{"type":14,"tag":35,"props":639,"children":641},{"id":640},"parallel-steps",[642],{"type":20,"value":643},"Parallel Steps",{"type":14,"tag":22,"props":645,"children":646},{},[647],{"type":20,"value":648},"Steps run simultaneously:",{"type":14,"tag":174,"props":650,"children":652},{"code":651},"Step 1: Extract from source A\nStep 2: Extract from source B  (both run at same time)\n   ↓\nStep 3: Combine and transform\n",[653],{"type":14,"tag":181,"props":654,"children":655},{"__ignoreMap":7},[656],{"type":20,"value":651},{"type":14,"tag":28,"props":658,"children":660},{"id":659},"common-workflows",[661],{"type":20,"value":662},"Common Workflows",{"type":14,"tag":35,"props":664,"children":666},{"id":665},"daily-report-generation",[667],{"type":20,"value":668},"Daily Report Generation",{"type":14,"tag":174,"props":670,"children":672},{"code":671},"Every day at 9 AM:\n1. Query sales data\n2. Calculate totals and trends\n3. Generate visualizations\n4. Send to stakeholders via email\n",[673],{"type":14,"tag":181,"props":674,"children":675},{"__ignoreMap":7},[676],{"type":20,"value":671},{"type":14,"tag":35,"props":678,"children":680},{"id":679},"data-quality-checks",[681],{"type":20,"value":682},"Data Quality Checks",{"type":14,"tag":174,"props":684,"children":686},{"code":685},"Every 6 hours:\n1. Count records in each table\n2. Check for null values\n3. Verify data types\n4. Alert if anomalies found\n",[687],{"type":14,"tag":181,"props":688,"children":689},{"__ignoreMap":7},[690],{"type":20,"value":685},{"type":14,"tag":35,"props":692,"children":694},{"id":693},"etl-pipeline",[695],{"type":20,"value":696},"ETL Pipeline",{"type":14,"tag":174,"props":698,"children":700},{"code":699},"Every night at 2 AM:\n1. Extract from external API\n2. Transform to standard format\n3. Load to database\n4. Update reports\n",[701],{"type":14,"tag":181,"props":702,"children":703},{"__ignoreMap":7},[704],{"type":20,"value":699},{"type":14,"tag":28,"props":706,"children":708},{"id":707},"advanced-features",[709],{"type":20,"value":710},"Advanced Features",{"type":14,"tag":35,"props":712,"children":714},{"id":713},"notifications",[715],{"type":20,"value":716},"Notifications",{"type":14,"tag":22,"props":718,"children":719},{},[720],{"type":20,"value":721},"Receive alerts:",{"type":14,"tag":47,"props":723,"children":724},{},[725,730,735,740],{"type":14,"tag":51,"props":726,"children":727},{},[728],{"type":20,"value":729},"Email on success\u002Ffailure",{"type":14,"tag":51,"props":731,"children":732},{},[733],{"type":20,"value":734},"Slack messages",{"type":14,"tag":51,"props":736,"children":737},{},[738],{"type":20,"value":739},"PagerDuty alerts",{"type":14,"tag":51,"props":741,"children":742},{},[743],{"type":20,"value":744},"Custom webhooks",{"type":14,"tag":35,"props":746,"children":748},{"id":747},"resource-limits",[749],{"type":20,"value":750},"Resource Limits",{"type":14,"tag":22,"props":752,"children":753},{},[754],{"type":20,"value":755},"Prevent runaway jobs:",{"type":14,"tag":174,"props":757,"children":759},{"code":758},"Max duration: 1 hour\nMax memory: 4 GB\nMax concurrent runs: 3\n",[760],{"type":14,"tag":181,"props":761,"children":762},{"__ignoreMap":7},[763],{"type":20,"value":758},{"type":14,"tag":35,"props":765,"children":767},{"id":766},"versioning",[768],{"type":20,"value":769},"Versioning",{"type":14,"tag":22,"props":771,"children":772},{},[773],{"type":20,"value":774},"Track pipeline changes:",{"type":14,"tag":47,"props":776,"children":777},{},[778,783,788,793],{"type":14,"tag":51,"props":779,"children":780},{},[781],{"type":20,"value":782},"Save versions",{"type":14,"tag":51,"props":784,"children":785},{},[786],{"type":20,"value":787},"Compare versions",{"type":14,"tag":51,"props":789,"children":790},{},[791],{"type":20,"value":792},"Rollback to previous",{"type":14,"tag":51,"props":794,"children":795},{},[796],{"type":20,"value":797},"View change history",{"type":14,"tag":28,"props":799,"children":801},{"id":800},"best-practices",[802],{"type":20,"value":803},"Best Practices",{"type":14,"tag":35,"props":805,"children":807},{"id":806},"pipeline-design",[808],{"type":20,"value":809},"Pipeline Design",{"type":14,"tag":47,"props":811,"children":812},{},[813,818,823,828],{"type":14,"tag":51,"props":814,"children":815},{},[816],{"type":20,"value":817},"Keep steps focused and single-purpose",{"type":14,"tag":51,"props":819,"children":820},{},[821],{"type":20,"value":822},"Use meaningful step names",{"type":14,"tag":51,"props":824,"children":825},{},[826],{"type":20,"value":827},"Add comments explaining complex logic",{"type":14,"tag":51,"props":829,"children":830},{},[831],{"type":20,"value":832},"Test with small datasets first",{"type":14,"tag":35,"props":834,"children":836},{"id":835},"monitoring",[837],{"type":20,"value":838},"Monitoring",{"type":14,"tag":47,"props":840,"children":841},{},[842,847,852,857],{"type":14,"tag":51,"props":843,"children":844},{},[845],{"type":20,"value":846},"Set up alerts for failures",{"type":14,"tag":51,"props":848,"children":849},{},[850],{"type":20,"value":851},"Review logs regularly",{"type":14,"tag":51,"props":853,"children":854},{},[855],{"type":20,"value":856},"Monitor execution time trends",{"type":14,"tag":51,"props":858,"children":859},{},[860],{"type":20,"value":861},"Track resource usage",{"type":14,"tag":35,"props":863,"children":865},{"id":864},"maintenance",[866],{"type":20,"value":867},"Maintenance",{"type":14,"tag":47,"props":869,"children":870},{},[871,876,881,886],{"type":14,"tag":51,"props":872,"children":873},{},[874],{"type":20,"value":875},"Archive old pipelines",{"type":14,"tag":51,"props":877,"children":878},{},[879],{"type":20,"value":880},"Update schedules seasonally",{"type":14,"tag":51,"props":882,"children":883},{},[884],{"type":20,"value":885},"Review and optimize slow steps",{"type":14,"tag":51,"props":887,"children":888},{},[889],{"type":20,"value":890},"Keep error handling up to date",{"type":14,"tag":28,"props":892,"children":894},{"id":893},"troubleshooting",[895],{"type":20,"value":896},"Troubleshooting",{"type":14,"tag":35,"props":898,"children":900},{"id":899},"pipeline-runs-too-long",[901],{"type":20,"value":902},"Pipeline runs too long",{"type":14,"tag":83,"props":904,"children":905},{},[906,911,916],{"type":14,"tag":51,"props":907,"children":908},{},[909],{"type":20,"value":910},"Check individual step times",{"type":14,"tag":51,"props":912,"children":913},{},[914],{"type":20,"value":915},"Identify slowest step",{"type":14,"tag":51,"props":917,"children":918},{},[919,921],{"type":20,"value":920},"Optimize that step:\n",{"type":14,"tag":47,"props":922,"children":923},{},[924,929,934],{"type":14,"tag":51,"props":925,"children":926},{},[927],{"type":20,"value":928},"Add database indexes",{"type":14,"tag":51,"props":930,"children":931},{},[932],{"type":20,"value":933},"Reduce dataset size",{"type":14,"tag":51,"props":935,"children":936},{},[937],{"type":20,"value":938},"Parallelize if possible",{"type":14,"tag":35,"props":940,"children":942},{"id":941},"out-of-memory-errors",[943],{"type":20,"value":944},"Out of memory errors",{"type":14,"tag":83,"props":946,"children":947},{},[948,953,958,963],{"type":14,"tag":51,"props":949,"children":950},{},[951],{"type":20,"value":952},"Process data in batches",{"type":14,"tag":51,"props":954,"children":955},{},[956],{"type":20,"value":957},"Delete intermediate results",{"type":14,"tag":51,"props":959,"children":960},{},[961],{"type":20,"value":962},"Use streaming where possible",{"type":14,"tag":51,"props":964,"children":965},{},[966],{"type":20,"value":967},"Increase resource limits",{"type":14,"tag":35,"props":969,"children":971},{"id":970},"jobs-not-running-on-schedule",[972],{"type":20,"value":973},"Jobs not running on schedule",{"type":14,"tag":83,"props":975,"children":976},{},[977,982,987,992],{"type":14,"tag":51,"props":978,"children":979},{},[980],{"type":20,"value":981},"Verify schedule is correct",{"type":14,"tag":51,"props":983,"children":984},{},[985],{"type":20,"value":986},"Check timezone settings",{"type":14,"tag":51,"props":988,"children":989},{},[990],{"type":20,"value":991},"Look for previous failed run blocking next",{"type":14,"tag":51,"props":993,"children":994},{},[995],{"type":20,"value":996},"Restart orchestration service if needed",{"type":14,"tag":28,"props":998,"children":1000},{"id":999},"related-topics",[1001],{"type":20,"value":1002},"Related Topics",{"type":14,"tag":47,"props":1004,"children":1005},{},[1006,1018,1029],{"type":14,"tag":51,"props":1007,"children":1008},{},[1009,1016],{"type":14,"tag":1010,"props":1011,"children":1013},"a",{"href":1012},"\u002Fdocs\u002Ffeatures\u002Flineage",[1014],{"type":20,"value":1015},"Lineage",{"type":20,"value":1017}," - Track data flow through pipelines",{"type":14,"tag":51,"props":1019,"children":1020},{},[1021,1027],{"type":14,"tag":1010,"props":1022,"children":1024},{"href":1023},"\u002Fdocs\u002Ffeatures\u002Factivity-logs",[1025],{"type":20,"value":1026},"Activity Logs",{"type":20,"value":1028}," - Audit pipeline executions",{"type":14,"tag":51,"props":1030,"children":1031},{},[1032,1038],{"type":14,"tag":1010,"props":1033,"children":1035},{"href":1034},"\u002Fdocs\u002Ffeatures\u002Fnotebook",[1036],{"type":20,"value":1037},"Notebook",{"type":20,"value":1039}," - Test pipeline steps",{"type":14,"tag":1041,"props":1042,"children":1043},"style",{},[1044],{"type":20,"value":1045},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":7,"searchDepth":197,"depth":197,"links":1047},[1048,1052,1056,1060,1065,1069,1073,1078,1083,1088,1093],{"id":30,"depth":197,"text":33,"children":1049},[1050,1051],{"id":37,"depth":206,"text":40},{"id":78,"depth":206,"text":81},{"id":119,"depth":197,"text":122,"children":1053},[1054,1055],{"id":125,"depth":206,"text":128},{"id":159,"depth":206,"text":162},{"id":351,"depth":197,"text":354,"children":1057},[1058,1059],{"id":357,"depth":206,"text":360},{"id":376,"depth":206,"text":379},{"id":430,"depth":197,"text":433,"children":1061},[1062,1063,1064],{"id":436,"depth":206,"text":439},{"id":475,"depth":206,"text":478},{"id":494,"depth":206,"text":497},{"id":523,"depth":197,"text":526,"children":1066},[1067,1068],{"id":529,"depth":206,"text":532},{"id":548,"depth":206,"text":551},{"id":615,"depth":197,"text":618,"children":1070},[1071,1072],{"id":621,"depth":206,"text":624},{"id":640,"depth":206,"text":643},{"id":659,"depth":197,"text":662,"children":1074},[1075,1076,1077],{"id":665,"depth":206,"text":668},{"id":679,"depth":206,"text":682},{"id":693,"depth":206,"text":696},{"id":707,"depth":197,"text":710,"children":1079},[1080,1081,1082],{"id":713,"depth":206,"text":716},{"id":747,"depth":206,"text":750},{"id":766,"depth":206,"text":769},{"id":800,"depth":197,"text":803,"children":1084},[1085,1086,1087],{"id":806,"depth":206,"text":809},{"id":835,"depth":206,"text":838},{"id":864,"depth":206,"text":867},{"id":893,"depth":197,"text":896,"children":1089},[1090,1091,1092],{"id":899,"depth":206,"text":902},{"id":941,"depth":206,"text":944},{"id":970,"depth":206,"text":973},{"id":999,"depth":197,"text":1002},"markdown","content:docs:features:orchestration.md","content","docs\u002Ffeatures\u002Forchestration.md","docs\u002Ffeatures\u002Forchestration","md",1782233761657]