{ "api_version": "1", "kind": "dse_post", "self": "https://update.dsesecurity.com/api/v1/posts/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/", "item": { "id": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/", "slug": "dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline", "url": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/", "alternate_urls": { "markdown": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline.md", "json": "https://update.dsesecurity.com/api/v1/posts/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/" }, "title": "Review file granularity before scaling a Data Lake analytics pipeline", "summary": "Many small input files can impose repeated metadata work; evaluate aggregation against downstream query needs.", "format": { "slug": "guide", "name": "Guide" }, "priority": { "slug": "info", "name": "Information" }, "featured": false, "image": { "theme": "network-infrastructure", "label": "Networks & infrastructure", "alt": "Resilient network core with engineered blue and gold data paths.", "card_url": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-card.webp?v=1.8.20", "hero_url": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-hero.webp?v=1.8.20", "social_url": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-social-v2.jpg?v=1.8.20", "width": 2400, "height": 1350 }, "topics": [ { "slug": "cybersecurity", "name": "Cybersecurity", "url": "https://update.dsesecurity.com/topic/cybersecurity/" }, { "slug": "networks-infrastructure", "name": "Networks & Infrastructure", "url": "https://update.dsesecurity.com/topic/networks-infrastructure/" } ], "author": { "name": "DSE Security Editorial Team", "url": "https://update.dsesecurity.com/#editorial-team", "type": "Organization" }, "publisher": { "name": "Detection Systems & Engineering", "url": "https://dsesecurity.com/" }, "published_at": "2026-09-10T00:27:41+00:00", "modified_at": "2026-09-10T01:23:48+00:00", "reviewed_on": "2026-09-09", "reading_minutes": 2, "word_count": 235, "potentially_affected": "Analytics pipelines reading many small files from Azure Data Lake Storage-enabled accounts.", "dse_recommendation": "Measure per-file overhead and test a larger-file, read-oriented layout before changing the production pipeline.", "primary_source": { "name": "Best Practices for Using Azure Data Lake Storage - Azure Storage | Microsoft Learn", "url": "https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-best-practices", "published_on": null, "authority": "Microsoft Learn" }, "publishing_principles": "https://update.dsesecurity.com/updates/dse-updates-editorial-methodology/", "usage_info": "https://update.dsesecurity.com/usage/", "copyright_notice": "Copyright © 2026 Detection Systems & Engineering. All rights reserved.", "content_html": "

Source facts

\n

Microsoft’s Data Lake guidance identifies per-file work such as listing, access checks and metadata operations as overhead for analytics engines. Many small files can therefore hurt processing performance.

\n

Where raw inputs cannot be controlled, the guidance recommends aggregating them into larger downstream files and considering a read-oriented format such as Parquet. It also warns that some engines process files above 100 GB inefficiently, so larger is not an unlimited prescription. Microsoft Learn.

\n

Applicability

\n

Identify the raw arrival pattern, downstream engine, query shape and current file-size distribution. Keep raw capture requirements separate from the layout used for repeated analysis.

\n

DSE recommendation

\n

DSE recommends a representative aggregation experiment with explicit freshness and correctness requirements. Choose the target size and format with the downstream owner rather than copying a universal setting. Preserve a reconciliation path from the aggregated output back to the original inputs. Include late arrivals and failed aggregation runs in the design before replacing a production consumer’s input path.

\n

Verification

\n

Compare the same approved dataset and queries before and after the layout change. Record file count, processing duration, resource use and any missing or repeated records. Check that the consumer’s required freshness remains acceptable. Approve the change only when measured improvement and data reconciliation support it; a lower file count alone is not evidence that the resulting dataset is correct.

\n

Official references

\n

Microsoft Learn: Best Practices for Using Azure Data Lake Storage. Source retrieved September 9, 2026.

", "content_text": "Source facts\nMicrosoft’s Data Lake guidance identifies per-file work such as listing, access checks and metadata operations as overhead for analytics engines. Many small files can therefore hurt processing performance.\nWhere raw inputs cannot be controlled, the guidance recommends aggregating them into larger downstream files and considering a read-oriented format such as Parquet. It also warns that some engines process files above 100 GB inefficiently, so larger is not an unlimited prescription. Microsoft Learn.\nApplicability\nIdentify the raw arrival pattern, downstream engine, query shape and current file-size distribution. Keep raw capture requirements separate from the layout used for repeated analysis.\nDSE recommendation\nDSE recommends a representative aggregation experiment with explicit freshness and correctness requirements. Choose the target size and format with the downstream owner rather than copying a universal setting. Preserve a reconciliation path from the aggregated output back to the original inputs. Include late arrivals and failed aggregation runs in the design before replacing a production consumer’s input path.\nVerification\nCompare the same approved dataset and queries before and after the layout change. Record file count, processing duration, resource use and any missing or repeated records. Check that the consumer’s required freshness remains acceptable. Approve the change only when measured improvement and data reconciliation support it; a lower file count alone is not evidence that the resulting dataset is correct.\nOfficial references\nMicrosoft Learn: Best Practices for Using Azure Data Lake Storage. Source retrieved September 9, 2026.", "content_markdown": "## Source facts\n\nMicrosoft’s Data Lake guidance identifies per-file work such as listing, access checks and metadata operations as overhead for analytics engines. Many small files can therefore hurt processing performance.\n\nWhere raw inputs cannot be controlled, the guidance recommends aggregating them into larger downstream files and considering a read-oriented format such as Parquet. It also warns that some engines process files above 100 GB inefficiently, so larger is not an unlimited prescription. [Microsoft Learn](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-best-practices).\n\n## Applicability\n\nIdentify the raw arrival pattern, downstream engine, query shape and current file-size distribution. Keep raw capture requirements separate from the layout used for repeated analysis.\n\n## DSE recommendation\n\nDSE recommends a representative aggregation experiment with explicit freshness and correctness requirements. Choose the target size and format with the downstream owner rather than copying a universal setting. Preserve a reconciliation path from the aggregated output back to the original inputs. Include late arrivals and failed aggregation runs in the design before replacing a production consumer’s input path.\n\n## Verification\n\nCompare the same approved dataset and queries before and after the layout change. Record file count, processing duration, resource use and any missing or repeated records. Check that the consumer’s required freshness remains acceptable. Approve the change only when measured improvement and data reconciliation support it; a lower file count alone is not evidence that the resulting dataset is correct.\n\n## Official references\n\n[Microsoft Learn: Best Practices for Using Azure Data Lake Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-best-practices). Source retrieved September 9, 2026." }, "json_ld": { "@context": "https://schema.org", "@graph": [ { "@type": "Organization", "@id": "https://dsesecurity.com/#organization", "name": "Detection Systems & Engineering", "alternateName": "DSE Security", "url": "https://dsesecurity.com/", "logo": { "@type": "ImageObject", "url": "https://update.dsesecurity.com/assets/dse-logo-20260812.png?v=1.8.20" } }, { "@type": "Organization", "@id": "https://update.dsesecurity.com/#editorial-team", "name": "DSE Security Editorial Team", "url": "https://update.dsesecurity.com/", "parentOrganization": { "@id": "https://dsesecurity.com/#organization" } }, { "@type": "WebSite", "@id": "https://update.dsesecurity.com/#website", "name": "DSE Updates", "alternateName": "DSE Security Knowledge Hub", "url": "https://update.dsesecurity.com/", "inLanguage": "en-US", "publisher": { "@id": "https://dsesecurity.com/#organization" }, "potentialAction": { "@type": "SearchAction", "target": { "@type": "EntryPoint", "urlTemplate": "https://update.dsesecurity.com/?q={search_term_string}" }, "query-input": "required name=search_term_string" } }, { "@type": "WebPage", "@id": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/", "url": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/", "isPartOf": { "@id": "https://update.dsesecurity.com/#website" }, "lastReviewed": "2026-09-09" }, { "@type": "BreadcrumbList", "@id": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/#breadcrumbs", "itemListElement": [ { "@type": "ListItem", "position": 1, "name": "DSE Updates", "item": "https://update.dsesecurity.com/" }, { "@type": "ListItem", "position": 2, "name": "Review file granularity before scaling a Data Lake analytics pipeline", "item": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/" } ] }, { "@type": [ "Article", "TechArticle" ], "@id": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/#article", "identifier": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/", "url": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/", "headline": "Review file granularity before scaling a Data Lake analytics pipeline", "description": "Many small input files can impose repeated metadata work; evaluate aggregation against downstream query needs.", "abstract": "Many small input files can impose repeated metadata work; evaluate aggregation against downstream query needs.", "articleBody": "Source facts\nMicrosoft’s Data Lake guidance identifies per-file work such as listing, access checks and metadata operations as overhead for analytics engines. Many small files can therefore hurt processing performance.\nWhere raw inputs cannot be controlled, the guidance recommends aggregating them into larger downstream files and considering a read-oriented format such as Parquet. It also warns that some engines process files above 100 GB inefficiently, so larger is not an unlimited prescription. Microsoft Learn.\nApplicability\nIdentify the raw arrival pattern, downstream engine, query shape and current file-size distribution. Keep raw capture requirements separate from the layout used for repeated analysis.\nDSE recommendation\nDSE recommends a representative aggregation experiment with explicit freshness and correctness requirements. Choose the target size and format with the downstream owner rather than copying a universal setting. Preserve a reconciliation path from the aggregated output back to the original inputs. Include late arrivals and failed aggregation runs in the design before replacing a production consumer’s input path.\nVerification\nCompare the same approved dataset and queries before and after the layout change. Record file count, processing duration, resource use and any missing or repeated records. Check that the consumer’s required freshness remains acceptable. Approve the change only when measured improvement and data reconciliation support it; a lower file count alone is not evidence that the resulting dataset is correct.\nOfficial references\nMicrosoft Learn: Best Practices for Using Azure Data Lake Storage. Source retrieved September 9, 2026.", "datePublished": "2026-09-10T00:27:41+00:00", "dateModified": "2026-09-10T01:23:48+00:00", "mainEntityOfPage": { "@id": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/" }, "inLanguage": "en-US", "isAccessibleForFree": true, "author": { "@type": "Organization", "name": "DSE Security Editorial Team", "url": "https://update.dsesecurity.com/#editorial-team" }, "publisher": { "@id": "https://dsesecurity.com/#organization" }, "image": { "@type": "ImageObject", "@id": "https://update.dsesecurity.com/updates/dse-20260909-255-review-file-granularity-before-scaling-a-data-lake-analytics-pipeline/#primaryimage", "url": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-social-v2.jpg?v=1.8.20", "contentUrl": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-social-v2.jpg?v=1.8.20", "width": 1200, "height": 630, "caption": "Review file granularity before scaling a Data Lake analytics pipeline" }, "articleSection": [ "Cybersecurity", "Networks & Infrastructure" ], "keywords": [ "Cybersecurity", "Networks & Infrastructure", "Guide", "Information priority" ], "genre": "Guide", "about": [ { "@type": "Thing", "name": "Cybersecurity", "url": "https://update.dsesecurity.com/topic/cybersecurity/" }, { "@type": "Thing", "name": "Networks & Infrastructure", "url": "https://update.dsesecurity.com/topic/networks-infrastructure/" } ], "wordCount": 235, "timeRequired": "PT2M", "publishingPrinciples": "https://update.dsesecurity.com/updates/dse-updates-editorial-methodology/", "usageInfo": "https://update.dsesecurity.com/usage/", "copyrightHolder": { "@id": "https://dsesecurity.com/#organization" }, "copyrightNotice": "Copyright © 2026 Detection Systems & Engineering. All rights reserved.", "citation": { "@type": "CreativeWork", "name": "Best Practices for Using Azure Data Lake Storage - Azure Storage | Microsoft Learn", "url": "https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-best-practices" } } ] } }