diff --git a/docs/reference-guides/core-blocks.md b/docs/reference-guides/core-blocks.md
index 0715b1e3547e2..b82963ec40d19 100644
--- a/docs/reference-guides/core-blocks.md
+++ b/docs/reference-guides/core-blocks.md
@@ -52,7 +52,7 @@ Prompt visitors to take action with a button-style link. ([Source](https://githu
 -	**Category:** design
 -	**Parent:** core/buttons
 -	**Supports:** anchor, color (background, gradients, text), interactivity (clientNavigation), shadow (), spacing (padding), splitting, typography (fontSize, lineHeight), ~~alignWide~~, ~~align~~, ~~reusable~~
--	**Attributes:** backgroundColor, gradient, linkTarget, placeholder, rel, tagName, text, textAlign, textColor, title, type, url, width
+-	**Attributes:** backgroundColor, download, gradient, linkTarget, placeholder, rel, tagName, text, textAlign, textColor, title, type, url, width
 
 ## Buttons
 
@@ -273,7 +273,7 @@ Add a link to a downloadable file. ([Source](https://github.com/WordPress/gutenb
 -	**Name:** core/file
 -	**Category:** media
 -	**Supports:** align, anchor, color (background, gradients, link, ~~text~~), interactivity, spacing (margin, padding)
--	**Attributes:** blob, displayPreview, downloadButtonText, fileId, fileName, href, id, previewHeight, showDownloadButton, textLinkHref, textLinkTarget
+-	**Attributes:** blob, displayPreview, fileId, fileName, href, id, previewHeight, showDownloadButton, textLinkHref, textLinkTarget
 
 ## Footnotes
 
diff --git a/packages/block-library/src/button/block.json b/packages/block-library/src/button/block.json
index 6fcb7aca4c592..3a6721f5a5d55 100644
--- a/packages/block-library/src/button/block.json
+++ b/packages/block-library/src/button/block.json
@@ -55,6 +55,13 @@
 			"attribute": "rel",
 			"role": "content"
 		},
+		"download": {
+			"type": "boolean",
+			"source": "attribute",
+			"selector": "a",
+			"attribute": "download",
+			"role": "content"
+		},
 		"placeholder": {
 			"type": "string"
 		},
diff --git a/packages/block-library/src/button/edit.js b/packages/block-library/src/button/edit.js
index 593066d6555b4..a656df43ad6cb 100644
--- a/packages/block-library/src/button/edit.js
+++ b/packages/block-library/src/button/edit.js
@@ -59,6 +59,10 @@ const LINK_SETTINGS = [
 		id: 'nofollow',
 		title: __( 'Mark as nofollow' ),
 	},
+	{
+		id: 'download',
+		title: __( 'Download file' ),
+	},
 ];
 
 function useEnter( props ) {
@@ -380,6 +384,7 @@ function ButtonEdit( props ) {
 								url: newURL,
 								opensInNewTab: newOpensInNewTab,
 								nofollow: newNofollow,
+								download: newDownload,
 							} ) =>
 								setAttributes(
 									getUpdatedLinkAttributes( {
@@ -387,6 +392,7 @@ function ButtonEdit( props ) {
 										url: newURL,
 										opensInNewTab: newOpensInNewTab,
 										nofollow: newNofollow,
+										download: newDownload,
 									} )
 								)
 							}
diff --git a/packages/block-library/src/button/get-updated-link-attributes.js b/packages/block-library/src/button/get-updated-link-attributes.js
index 0e39a1815a318..c0d81d1590c11 100644
--- a/packages/block-library/src/button/get-updated-link-attributes.js
+++ b/packages/block-library/src/button/get-updated-link-attributes.js
@@ -16,12 +16,14 @@ import { prependHTTP } from '@wordpress/url';
  * @param {string}  attributes.url           The current link url.
  * @param {boolean} attributes.opensInNewTab Whether the link should open in a new window.
  * @param {boolean} attributes.nofollow      Whether the link should be marked as nofollow.
+ * @param {boolean} attributes.download      Whether the link should allow download.
  */
 export function getUpdatedLinkAttributes( {
 	rel = '',
 	url = '',
 	opensInNewTab,
 	nofollow,
+	download = false,
 } ) {
 	let newLinkTarget;
 	// Since `rel` is editable attribute, we need to check for existing values and proceed accordingly.
@@ -46,9 +48,33 @@ export function getUpdatedLinkAttributes( {
 		updatedRel = updatedRel?.replace( relRegex, '' ).trim();
 	}
 
+	const allowDownload = url && isSameOrigin( url ) ? download : undefined;
+
 	return {
 		url: prependHTTP( url ),
 		linkTarget: newLinkTarget,
 		rel: updatedRel || undefined,
+		download: allowDownload,
 	};
 }
+
+/**
+ * Checks if the URL is same origin.
+ * Allow relative URLs.
+ *
+ * @param {string} urlString The URL to check.
+ * @return {boolean} Whether the URL is same origin.
+ */
+function isSameOrigin( urlString ) {
+	// Allow relative URLs
+	if ( urlString.startsWith( '/' ) ) {
+		return true;
+	}
+
+	try {
+		const url = new URL( urlString, window.location.origin );
+		return url.origin === window.location.origin;
+	} catch {
+		return false;
+	}
+}
diff --git a/packages/block-library/src/button/save.js b/packages/block-library/src/button/save.js
index 4255868d50fbc..dcd647ea29b18 100644
--- a/packages/block-library/src/button/save.js
+++ b/packages/block-library/src/button/save.js
@@ -30,6 +30,7 @@ export default function save( { attributes, className } ) {
 		title,
 		url,
 		width,
+		download,
 	} = attributes;
 
 	const TagName = tagName || 'a';
@@ -83,6 +84,7 @@ export default function save( { attributes, className } ) {
 				value={ text }
 				target={ isButtonTag ? null : linkTarget }
 				rel={ isButtonTag ? null : rel }
+				download={ isButtonTag ? null : download }
 			/>
 		</div>
 	);
diff --git a/packages/block-library/src/file/block.json b/packages/block-library/src/file/block.json
index 2c5e888c2aff6..ce887770cc3c1 100644
--- a/packages/block-library/src/file/block.json
+++ b/packages/block-library/src/file/block.json
@@ -48,12 +48,6 @@
 			"type": "boolean",
 			"default": true
 		},
-		"downloadButtonText": {
-			"type": "rich-text",
-			"source": "rich-text",
-			"selector": "a[download]",
-			"role": "content"
-		},
 		"displayPreview": {
 			"type": "boolean"
 		},
diff --git a/packages/block-library/src/file/edit.js b/packages/block-library/src/file/edit.js
index 838b807507d31..687ededf674ce 100644
--- a/packages/block-library/src/file/edit.js
+++ b/packages/block-library/src/file/edit.js
@@ -21,11 +21,11 @@ import {
 	RichText,
 	useBlockProps,
 	store as blockEditorStore,
-	__experimentalGetElementClassName,
+	useInnerBlocksProps,
 } from '@wordpress/block-editor';
-import { useEffect, useState } from '@wordpress/element';
+import { useState } from '@wordpress/element';
 import { useCopyToClipboard } from '@wordpress/compose';
-import { __, _x } from '@wordpress/i18n';
+import { __ } from '@wordpress/i18n';
 import { file as icon } from '@wordpress/icons';
 import { store as coreStore } from '@wordpress/core-data';
 import { store as noticesStore } from '@wordpress/notices';
@@ -69,7 +69,6 @@ function FileEdit( { attributes, isSelected, setAttributes, clientId } ) {
 		textLinkHref,
 		textLinkTarget,
 		showDownloadButton,
-		downloadButtonText,
 		displayPreview,
 		previewHeight,
 	} = attributes;
@@ -85,8 +84,7 @@ function FileEdit( { attributes, isSelected, setAttributes, clientId } ) {
 	);
 
 	const { createErrorNotice } = useDispatch( noticesStore );
-	const { toggleSelection, __unstableMarkNextChangeAsNotPersistent } =
-		useDispatch( blockEditorStore );
+	const { toggleSelection } = useDispatch( blockEditorStore );
 
 	useUploadMediaFromBlobURL( {
 		url: temporaryURL,
@@ -94,18 +92,6 @@ function FileEdit( { attributes, isSelected, setAttributes, clientId } ) {
 		onError: onUploadError,
 	} );
 
-	// Note: Handle setting a default value for `downloadButtonText` via HTML API
-	// when it supports replacing text content for HTML tags.
-	useEffect( () => {
-		if ( RichText.isEmpty( downloadButtonText ) ) {
-			__unstableMarkNextChangeAsNotPersistent();
-			setAttributes( {
-				downloadButtonText: _x( 'Download', 'button label' ),
-			} );
-		}
-		// This effect should only run on mount.
-	}, [] );
-
 	function onSelectFile( newMedia ) {
 		if ( ! newMedia || ! newMedia.url ) {
 			// Reset attributes.
@@ -199,6 +185,36 @@ function FileEdit( { attributes, isSelected, setAttributes, clientId } ) {
 
 	const displayPreviewInEditor = browserSupportsPdfs() && displayPreview;
 
+	const innerBlocksProps = useInnerBlocksProps(
+		{ className: 'wp-block-file__button-wrapper' },
+		{
+			allowedBlocks: [ 'core/buttons' ],
+			template: [
+				[
+					'core/buttons',
+					{},
+					[
+						[
+							'core/button',
+							{
+								text: __( 'Download' ),
+								lock: {
+									remove: true,
+									move: true,
+								},
+								url: href || temporaryURL,
+								download: true,
+								ariaLabel: __( 'Download button text' ),
+							},
+						],
+					],
+				],
+			],
+			templateLock: 'all',
+			renderAppender: false,
+		}
+	);
+
 	if ( ! href && ! temporaryURL ) {
 		return (
 			<div { ...blockProps }>
@@ -301,31 +317,7 @@ function FileEdit( { attributes, isSelected, setAttributes, clientId } ) {
 						}
 						href={ textLinkHref }
 					/>
-					{ showDownloadButton && (
-						<div className="wp-block-file__button-richtext-wrapper">
-							{ /* Using RichText here instead of PlainText so that it can be styled like a button. */ }
-							<RichText
-								identifier="downloadButtonText"
-								tagName="div" // Must be block-level or else cursor disappears.
-								aria-label={ __( 'Download button text' ) }
-								className={ clsx(
-									'wp-block-file__button',
-									__experimentalGetElementClassName(
-										'button'
-									)
-								) }
-								value={ downloadButtonText }
-								withoutInteractiveFormatting
-								placeholder={ __( 'Add text…' ) }
-								onChange={ ( text ) =>
-									setAttributes( {
-										downloadButtonText:
-											removeAnchorTag( text ),
-									} )
-								}
-							/>
-						</div>
-					) }
+					{ showDownloadButton && <div { ...innerBlocksProps } /> }
 				</div>
 			</div>
 		</>
diff --git a/packages/block-library/src/file/editor.scss b/packages/block-library/src/file/editor.scss
index 1a38662381447..c060bd075705a 100644
--- a/packages/block-library/src/file/editor.scss
+++ b/packages/block-library/src/file/editor.scss
@@ -33,7 +33,11 @@
 	}
 
 	.wp-block-file__content-wrapper {
-		flex-grow: 1;
+		display: flex;
+		align-items: center;
+		gap: 1em;
+		flex-wrap: wrap;
+		width: 100%;
 	}
 
 	a {
@@ -43,9 +47,4 @@
 			display: inline-block;
 		}
 	}
-
-	.wp-block-file__button-richtext-wrapper {
-		display: inline-block;
-		margin-left: 0.75em;
-	}
 }
diff --git a/packages/block-library/src/file/save.js b/packages/block-library/src/file/save.js
index 34b1046d6d2bb..17d09d2244352 100644
--- a/packages/block-library/src/file/save.js
+++ b/packages/block-library/src/file/save.js
@@ -1,15 +1,10 @@
-/**
- * External dependencies
- */
-import clsx from 'clsx';
-
 /**
  * WordPress dependencies
  */
 import {
 	RichText,
 	useBlockProps,
-	__experimentalGetElementClassName,
+	useInnerBlocksProps,
 } from '@wordpress/block-editor';
 
 export default function save( { attributes } ) {
@@ -20,7 +15,6 @@ export default function save( { attributes } ) {
 		textLinkHref,
 		textLinkTarget,
 		showDownloadButton,
-		downloadButtonText,
 		displayPreview,
 		previewHeight,
 	} = attributes;
@@ -37,9 +31,18 @@ export default function save( { attributes } ) {
 	// actually rendered.
 	const describedById = hasFilename ? fileId : undefined;
 
+	const blockProps = useBlockProps.save( {
+		className: 'wp-block-file',
+	} );
+
+	// Use the `useInnerBlocksProps` hook to get the props for the inner blocks
+	const innerBlocksProps = useInnerBlocksProps.save( {
+		className: 'wp-block-file__button-wrapper',
+	} );
+
 	return (
 		href && (
-			<div { ...useBlockProps.save() }>
+			<div { ...blockProps }>
 				{ displayPreview && (
 					<>
 						<object
@@ -66,19 +69,7 @@ export default function save( { attributes } ) {
 						<RichText.Content value={ fileName } />
 					</a>
 				) }
-				{ showDownloadButton && (
-					<a
-						href={ href }
-						className={ clsx(
-							'wp-block-file__button',
-							__experimentalGetElementClassName( 'button' )
-						) }
-						download
-						aria-describedby={ describedById }
-					>
-						<RichText.Content value={ downloadButtonText } />
-					</a>
-				) }
+				{ showDownloadButton && <div { ...innerBlocksProps } /> }
 			</div>
 		)
 	);
diff --git a/packages/block-library/src/file/style.scss b/packages/block-library/src/file/style.scss
index 31c76ff2641fb..71687a07f48fd 100644
--- a/packages/block-library/src/file/style.scss
+++ b/packages/block-library/src/file/style.scss
@@ -1,6 +1,10 @@
 .wp-block-file {
 	// This block has customizable padding, border-box makes that more predictable.
 	box-sizing: border-box;
+	display: flex;
+	flex-wrap: wrap;
+	align-items: center;
+	gap: 1em;
 
 	&:not(.wp-element-button) {
 		font-size: 0.8em;
@@ -18,6 +22,10 @@
 	* + .wp-block-file__button {
 		margin-left: 0.75em;
 	}
+
+	.wp-block-file__button-wrapper {
+		flex: 1;
+	}
 }
 
 // Lowest specificity to avoid overriding layout styles.
@@ -30,20 +38,20 @@
 }
 
 //This needs a low specificity so it won't override the rules from the button element if defined in theme.json.
-:where(.wp-block-file__button) {
-	border-radius: 2em;
-	padding: 0.5em 1em;
-	display: inline-block;
-
-	&:is(a) {
-		&:hover,
-		&:visited,
-		&:focus,
-		&:active {
-			box-shadow: none;
-			color: $white;
-			opacity: 0.85;
-			text-decoration: none;
-		}
-	}
-}
+// :where(.wp-block-file__button) {
+// 	border-radius: 2em;
+// 	padding: 0.5em 1em;
+// 	display: inline-block;
+
+// 	&:is(a) {
+// 		&:hover,
+// 		&:visited,
+// 		&:focus,
+// 		&:active {
+// 			box-shadow: none;
+// 			color: $white;
+// 			opacity: 0.85;
+// 			text-decoration: none;
+// 		}
+// 	}
+// }