{ "api_version": "1", "kind": "dse_post", "self": "https://update.dsesecurity.com/api/v1/posts/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/", "item": { "id": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/", "slug": "dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job", "url": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/", "alternate_urls": { "markdown": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job.md", "json": "https://update.dsesecurity.com/api/v1/posts/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/" }, "title": "Separate durable HPC storage from the layer that accelerates a job", "summary": "Which part of an HPC storage proposal is the dataset's permanent home, and is a separate accelerator justified?", "format": { "slug": "guide", "name": "Guide" }, "priority": { "slug": "info", "name": "Information" }, "featured": false, "image": { "theme": "network-infrastructure", "label": "Networks & infrastructure", "alt": "Resilient network core with engineered blue and gold data paths.", "card_url": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-card.webp?v=1.8.20", "hero_url": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-hero.webp?v=1.8.20", "social_url": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-social-v2.jpg?v=1.8.20", "width": 2400, "height": 1350 }, "topics": [ { "slug": "it", "name": "IT", "url": "https://update.dsesecurity.com/topic/it/" }, { "slug": "networks-infrastructure", "name": "Networks & Infrastructure", "url": "https://update.dsesecurity.com/topic/networks-infrastructure/" } ], "author": { "name": "DSE Security Editorial Team", "url": "https://update.dsesecurity.com/#editorial-team", "type": "Organization" }, "publisher": { "name": "Detection Systems & Engineering", "url": "https://dsesecurity.com/" }, "published_at": "2026-09-10T00:30:13+00:00", "modified_at": "2026-09-10T00:52:38+00:00", "reviewed_on": "2026-09-09", "reading_minutes": 2, "word_count": 241, "potentially_affected": "Teams selecting storage for high-performance computing workloads on Azure VMs.", "dse_recommendation": "Document the durable dataset location separately from any proposed performance accelerator, then qualify the combination with the workload's file profile.", "primary_source": { "name": "Azure HPC workload best practices guide - Azure Virtual Machines | Microsoft Learn", "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/workload-guidelines-best-practices-storage", "published_on": null, "authority": "Microsoft Learn" }, "publishing_principles": "https://update.dsesecurity.com/updates/dse-updates-editorial-methodology/", "usage_info": "https://update.dsesecurity.com/usage/", "copyright_notice": "Copyright © 2026 Detection Systems & Engineering. All rights reserved.", "content_html": "

Source facts

\n

Microsoft distinguishes an HPC dataset’s permanent storage from an optional acceleration layer that improves computational access. Its selection guide considers stored volume, CPU count, and file sizes together. For datasets below 50 TiB, the guide proposes Azure Files or Azure NetApp Files without an accelerator; it also gives that recommendation for 50–5,000 TiB with fewer than 500 cores. These are workload-selection guidelines, not a measured result for a particular application. Microsoft Learn.

\n

Applicability

\n

Use this distinction when reviewing an Azure VM HPC storage design. Describe the actual job’s dataset and access requirements before adopting a diagram intended for a different scale. Do not substitute a service name for a statement of which layer owns the durable copy.

\n

DSE recommendation

\n

Document the durable dataset location separately from any proposed performance accelerator, then qualify the combination with the workload’s file profile. Ask the designer to justify an accelerator against a simpler core-storage-only proposal. Include input staging, intermediate output, and final result placement in that comparison. Treat the source’s branches as starting points for evaluation, not mandatory architecture for every job.

\n

Verification

\n

Rehearse a representative job using the proposed arrangement. Record its file-size distribution, core count, data volume, and observed access behavior. Confirm where final results are retained before any temporary resources are removed. Keep a failed throughput target separate from a failed persistence requirement so the next design change addresses the actual gap.

\n

Official references

\n

Microsoft Learn: HPC storage best practices. Source reviewed September 9, 2026.

", "content_text": "Source facts\nMicrosoft distinguishes an HPC dataset’s permanent storage from an optional acceleration layer that improves computational access. Its selection guide considers stored volume, CPU count, and file sizes together. For datasets below 50 TiB, the guide proposes Azure Files or Azure NetApp Files without an accelerator; it also gives that recommendation for 50–5,000 TiB with fewer than 500 cores. These are workload-selection guidelines, not a measured result for a particular application. Microsoft Learn.\nApplicability\nUse this distinction when reviewing an Azure VM HPC storage design. Describe the actual job’s dataset and access requirements before adopting a diagram intended for a different scale. Do not substitute a service name for a statement of which layer owns the durable copy.\nDSE recommendation\nDocument the durable dataset location separately from any proposed performance accelerator, then qualify the combination with the workload’s file profile. Ask the designer to justify an accelerator against a simpler core-storage-only proposal. Include input staging, intermediate output, and final result placement in that comparison. Treat the source’s branches as starting points for evaluation, not mandatory architecture for every job.\nVerification\nRehearse a representative job using the proposed arrangement. Record its file-size distribution, core count, data volume, and observed access behavior. Confirm where final results are retained before any temporary resources are removed. Keep a failed throughput target separate from a failed persistence requirement so the next design change addresses the actual gap.\nOfficial references\nMicrosoft Learn: HPC storage best practices. Source reviewed September 9, 2026.", "content_markdown": "## Source facts\n\nMicrosoft distinguishes an HPC dataset’s permanent storage from an optional acceleration layer that improves computational access. Its selection guide considers stored volume, CPU count, and file sizes together. For datasets below 50 TiB, the guide proposes Azure Files or Azure NetApp Files without an accelerator; it also gives that recommendation for 50–5,000 TiB with fewer than 500 cores. These are workload-selection guidelines, not a measured result for a particular application. [Microsoft Learn](https://learn.microsoft.com/en-us/azure/virtual-machines/workload-guidelines-best-practices-storage).\n\n## Applicability\n\nUse this distinction when reviewing an Azure VM HPC storage design. Describe the actual job’s dataset and access requirements before adopting a diagram intended for a different scale. Do not substitute a service name for a statement of which layer owns the durable copy.\n\n## DSE recommendation\n\nDocument the durable dataset location separately from any proposed performance accelerator, then qualify the combination with the workload’s file profile. Ask the designer to justify an accelerator against a simpler core-storage-only proposal. Include input staging, intermediate output, and final result placement in that comparison. Treat the source’s branches as starting points for evaluation, not mandatory architecture for every job.\n\n## Verification\n\nRehearse a representative job using the proposed arrangement. Record its file-size distribution, core count, data volume, and observed access behavior. Confirm where final results are retained before any temporary resources are removed. Keep a failed throughput target separate from a failed persistence requirement so the next design change addresses the actual gap.\n\n## Official references\n\n[Microsoft Learn: HPC storage best practices](https://learn.microsoft.com/en-us/azure/virtual-machines/workload-guidelines-best-practices-storage). Source reviewed September 9, 2026." }, "json_ld": { "@context": "https://schema.org", "@graph": [ { "@type": "Organization", "@id": "https://dsesecurity.com/#organization", "name": "Detection Systems & Engineering", "alternateName": "DSE Security", "url": "https://dsesecurity.com/", "logo": { "@type": "ImageObject", "url": "https://update.dsesecurity.com/assets/dse-logo-20260812.png?v=1.8.20" } }, { "@type": "Organization", "@id": "https://update.dsesecurity.com/#editorial-team", "name": "DSE Security Editorial Team", "url": "https://update.dsesecurity.com/", "parentOrganization": { "@id": "https://dsesecurity.com/#organization" } }, { "@type": "WebSite", "@id": "https://update.dsesecurity.com/#website", "name": "DSE Updates", "alternateName": "DSE Security Knowledge Hub", "url": "https://update.dsesecurity.com/", "inLanguage": "en-US", "publisher": { "@id": "https://dsesecurity.com/#organization" }, "potentialAction": { "@type": "SearchAction", "target": { "@type": "EntryPoint", "urlTemplate": "https://update.dsesecurity.com/?q={search_term_string}" }, "query-input": "required name=search_term_string" } }, { "@type": "WebPage", "@id": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/", "url": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/", "isPartOf": { "@id": "https://update.dsesecurity.com/#website" }, "lastReviewed": "2026-09-09" }, { "@type": "BreadcrumbList", "@id": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/#breadcrumbs", "itemListElement": [ { "@type": "ListItem", "position": 1, "name": "DSE Updates", "item": "https://update.dsesecurity.com/" }, { "@type": "ListItem", "position": 2, "name": "Separate durable HPC storage from the layer that accelerates a job", "item": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/" } ] }, { "@type": [ "Article", "TechArticle" ], "@id": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/#article", "identifier": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/", "url": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/", "headline": "Separate durable HPC storage from the layer that accelerates a job", "description": "Which part of an HPC storage proposal is the dataset's permanent home, and is a separate accelerator justified?", "abstract": "Which part of an HPC storage proposal is the dataset's permanent home, and is a separate accelerator justified?", "articleBody": "Source facts\nMicrosoft distinguishes an HPC dataset’s permanent storage from an optional acceleration layer that improves computational access. Its selection guide considers stored volume, CPU count, and file sizes together. For datasets below 50 TiB, the guide proposes Azure Files or Azure NetApp Files without an accelerator; it also gives that recommendation for 50–5,000 TiB with fewer than 500 cores. These are workload-selection guidelines, not a measured result for a particular application. Microsoft Learn.\nApplicability\nUse this distinction when reviewing an Azure VM HPC storage design. Describe the actual job’s dataset and access requirements before adopting a diagram intended for a different scale. Do not substitute a service name for a statement of which layer owns the durable copy.\nDSE recommendation\nDocument the durable dataset location separately from any proposed performance accelerator, then qualify the combination with the workload’s file profile. Ask the designer to justify an accelerator against a simpler core-storage-only proposal. Include input staging, intermediate output, and final result placement in that comparison. Treat the source’s branches as starting points for evaluation, not mandatory architecture for every job.\nVerification\nRehearse a representative job using the proposed arrangement. Record its file-size distribution, core count, data volume, and observed access behavior. Confirm where final results are retained before any temporary resources are removed. Keep a failed throughput target separate from a failed persistence requirement so the next design change addresses the actual gap.\nOfficial references\nMicrosoft Learn: HPC storage best practices. Source reviewed September 9, 2026.", "datePublished": "2026-09-10T00:30:13+00:00", "dateModified": "2026-09-10T00:52:38+00:00", "mainEntityOfPage": { "@id": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/" }, "inLanguage": "en-US", "isAccessibleForFree": true, "author": { "@type": "Organization", "name": "DSE Security Editorial Team", "url": "https://update.dsesecurity.com/#editorial-team" }, "publisher": { "@id": "https://dsesecurity.com/#organization" }, "image": { "@type": "ImageObject", "@id": "https://update.dsesecurity.com/updates/dse-20260909-103-separate-durable-hpc-storage-from-the-layer-that-accelerates-a-job/#primaryimage", "url": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-social-v2.jpg?v=1.8.20", "contentUrl": "https://update.dsesecurity.com/assets/editorial/network-infrastructure-social-v2.jpg?v=1.8.20", "width": 1200, "height": 630, "caption": "Separate durable HPC storage from the layer that accelerates a job" }, "articleSection": [ "IT", "Networks & Infrastructure" ], "keywords": [ "IT", "Networks & Infrastructure", "Guide", "Information priority" ], "genre": "Guide", "about": [ { "@type": "Thing", "name": "IT", "url": "https://update.dsesecurity.com/topic/it/" }, { "@type": "Thing", "name": "Networks & Infrastructure", "url": "https://update.dsesecurity.com/topic/networks-infrastructure/" } ], "wordCount": 241, "timeRequired": "PT2M", "publishingPrinciples": "https://update.dsesecurity.com/updates/dse-updates-editorial-methodology/", "usageInfo": "https://update.dsesecurity.com/usage/", "copyrightHolder": { "@id": "https://dsesecurity.com/#organization" }, "copyrightNotice": "Copyright © 2026 Detection Systems & Engineering. All rights reserved.", "citation": { "@type": "CreativeWork", "name": "Azure HPC workload best practices guide - Azure Virtual Machines | Microsoft Learn", "url": "https://learn.microsoft.com/en-us/azure/virtual-machines/workload-guidelines-best-practices-storage" } } ] } }