fix: get external resource blocked (#12927)

## Fix
- Use `[Config].upload.skipSafeFetch` to allow specific external urls
- Use `[Config].upload.pasteURL.allowList` to allow specific external
urls

Documentation: [Uploading files from remote
urls](https://payloadcms.com/docs/upload/overview#uploading-files-from-remote-urls)

Fixes: #12876
Mentioned: #7037,
#12934
Source PR: #12622
Issue Trace:
1. [`allowList`
Added](8b7f2dd#diff-92acf7b8d30e447a791e37820136bcbf23c42f0358daca0fdea4e7b77f7d4bc9
)

2. [`allowList`
Removed](648c168#diff-92acf7b8d30e447a791e37820136bcbf23c42f0358daca0fdea4e7b77f7d4bc9)

Author: kendelljoseph

docs/upload/overview.mdx

@@ -109,6 +109,7 @@ _An asterisk denotes that an option is required._
 | **`mimeTypes`**                | Restrict mimeTypes in the file picker. Array of valid mimetypes or mimetype wildcards [More](#mimetypes)                                                                                                                                           |
 | **`pasteURL`**                 | Controls whether files can be uploaded from remote URLs by pasting them into the Upload field. **Enabled by default.** Accepts `false` to disable or an object with an `allowList` of valid remote URLs. [More](#uploading-files-from-remote-urls) |
 | **`resizeOptions`**            | An object passed to the the Sharp image library to resize the uploaded file. [More](https://sharp.pixelplumbing.com/api-resize)                                                                                                                    |
+| **`skipSafeFetch`**            | Set to an `allowList` to skip the safe fetch check when fetching external files. Set to `true` to skip the safe fetch for all documents in this collection. Defaults to `false`.                                                                   |
 | **`staticDir`**                | The folder directory to use to store media in. Can be either an absolute path or relative to the directory that contains your config. Defaults to your collection slug                                                                             |
 | **`trimOptions`**              | An object passed to the the Sharp image library to trim the uploaded file. [More](https://sharp.pixelplumbing.com/api-resize#trim)                                                                                                                 |
 | **`withMetadata`**             | If specified, appends metadata to the output image file. Accepts a boolean or a function that receives `metadata` and `req`, returning a boolean.                                                                                                  |
@@ -435,6 +436,24 @@ export const Media: CollectionConfig = {
 }
 ```
 
+You can also adjust server-side fetching at the upload level as well, this does not effect the `CORS` policy like the `pasteURL` option does, but it allows you to skip the safe fetch check for specific URLs.
+
+```
+import type { CollectionConfig } from 'payload'
+
+export const Media: CollectionConfig = {
+  slug: 'media',
+  upload: {
+    skipSafeFetch: [
+      {
+        hostname: 'example.com',
+        pathname: '/images/*',
+      },
+    ],
+  },
+}
+```
+
 ##### Accepted Values for `pasteURL`
 
 | Option          | Description                                                                                                                                                    |

packages/payload/src/uploads/getExternalFile.ts

@@ -2,6 +2,7 @@ import type { PayloadRequest } from '../types/index.js'
 import type { File, FileData, UploadConfig } from './types.js'
 
 import { APIError } from '../errors/index.js'
+import { isURLAllowed } from '../utilities/isURLAllowed.js'
 import { safeFetch } from './safeFetch.js'
 
 type Args = {
@@ -23,11 +24,35 @@ export const getExternalFile = async ({ data, req, uploadConfig }: Args): Promis
       ? uploadConfig.externalFileHeaderFilter(Object.fromEntries(new Headers(req.headers)))
       : { cookie: req.headers.get('cookie')! }
 
-    const res = await safeFetch(fileURL, {
-      credentials: 'include',
-      headers,
-      method: 'GET',
-    })
+    // Check if URL is allowed because of skipSafeFetch allowList
+    const skipSafeFetch: boolean =
+      uploadConfig.skipSafeFetch === true
+        ? uploadConfig.skipSafeFetch
+        : Array.isArray(uploadConfig.skipSafeFetch) &&
+          isURLAllowed(fileURL, uploadConfig.skipSafeFetch)
+
+    // Check if URL is allowed because of pasteURL allowList
+    const isAllowedPasteUrl: boolean | undefined =
+      uploadConfig.pasteURL &&
+      uploadConfig.pasteURL.allowList &&
+      isURLAllowed(fileURL, uploadConfig.pasteURL.allowList)
+
+    let res
+    if (skipSafeFetch || isAllowedPasteUrl) {
+      // Allowed
+      res = await fetch(fileURL, {
+        credentials: 'include',
+        headers,
+        method: 'GET',
+      })
+    } else {
+      // Default
+      res = await safeFetch(fileURL, {
+        credentials: 'include',
+        headers,
+        method: 'GET',
+      })
+    }
 
     if (!res.ok) {
       throw new APIError(`Failed to fetch file from ${fileURL}`, res.status)

packages/payload/src/uploads/types.ts

@@ -225,7 +225,8 @@ export type UploadConfig = {
   /**
    * Controls the behavior of pasting/uploading files from URLs.
    * If set to `false`, fetching from remote URLs is disabled.
-   * If an allowList is provided, server-side fetching will be enabled for specified URLs.
+   * If an `allowList` is provided, server-side fetching will be enabled for specified URLs.
+   *
    * @default true (client-side fetching enabled)
    */
   pasteURL?:
@@ -239,6 +240,11 @@ export type UploadConfig = {
    * @default undefined
    */
   resizeOptions?: ResizeOptions
+  /**
+   *  Skip safe fetch when using server-side fetching for external files from these URLs.
+   *  @default false
+   */
+  skipSafeFetch?: AllowList | boolean
   /**
    * The directory to serve static files from. Defaults to collection slug.
    * @default undefined

packages/plugin-cloud-storage/src/plugin.ts

@@ -1,6 +1,6 @@
 import type { Config } from 'payload'
 
-import type { PluginOptions } from './types.js'
+import type { AllowList, PluginOptions } from './types.js'
 
 import { getFields } from './fields/getFields.js'
 import { getAfterDeleteHook } from './hooks/afterDelete.js'
@@ -70,6 +70,47 @@ export const cloudStoragePlugin =
             })
           }
 
+          const getSkipSafeFetchSetting = (): AllowList | boolean => {
+            if (options.disablePayloadAccessControl) {
+              return true
+            }
+            const isBooleanTrueSkipSafeFetch =
+              typeof existingCollection.upload === 'object' &&
+              existingCollection.upload.skipSafeFetch === true
+
+            const isAllowListSkipSafeFetch =
+              typeof existingCollection.upload === 'object' &&
+              Array.isArray(existingCollection.upload.skipSafeFetch)
+
+            if (isBooleanTrueSkipSafeFetch) {
+              return true
+            } else if (isAllowListSkipSafeFetch) {
+              const existingSkipSafeFetch =
+                typeof existingCollection.upload === 'object' &&
+                Array.isArray(existingCollection.upload.skipSafeFetch)
+                  ? existingCollection.upload.skipSafeFetch
+                  : []
+
+              const hasExactLocalhostMatch = existingSkipSafeFetch.some((entry) => {
+                const entryKeys = Object.keys(entry)
+                return entryKeys.length === 1 && entry.hostname === 'localhost'
+              })
+
+              const localhostEntry =
+                process.env.NODE_ENV !== 'production' && !hasExactLocalhostMatch
+                  ? [{ hostname: 'localhost' }]
+                  : []
+
+              return [...existingSkipSafeFetch, ...localhostEntry]
+            }
+
+            if (process.env.NODE_ENV !== 'production') {
+              return [{ hostname: 'localhost' }]
+            }
+
+            return false
+          }
+
           return {
             ...existingCollection,
             fields,
@@ -92,6 +133,7 @@ export const cloudStoragePlugin =
                   ? options.disableLocalStorage
                   : true,
               handlers,
+              skipSafeFetch: getSkipSafeFetchSetting(),
             },
           }
         }

packages/plugin-cloud-storage/src/types.ts

@@ -81,6 +81,14 @@ export interface GeneratedAdapter {
 
 export type Adapter = (args: { collection: CollectionConfig; prefix?: string }) => GeneratedAdapter
 
+export type AllowList = Array<{
+  hostname: string
+  pathname?: string
+  port?: string
+  protocol?: 'http' | 'https'
+  search?: string
+}>
+
 export type GenerateFileURL = (args: {
   collection: CollectionConfig
   filename: string

test/uploads/config.ts

@@ -1,5 +1,3 @@
-/* eslint-disable no-restricted-exports */
-
 import type { CollectionSlug, File } from 'payload'
 
 import path from 'path'
@@ -33,6 +31,7 @@ import {
   reduceSlug,
   relationPreviewSlug,
   relationSlug,
+  skipSafeFetchMediaSlug,
   threeDimensionalSlug,
   unstoredMediaSlug,
   versionSlug,
@@ -429,6 +428,14 @@ export default buildConfigWithDefaults({
         staticDir: path.resolve(dirname, './media'),
       },
     },
+    {
+      slug: skipSafeFetchMediaSlug,
+      fields: [],
+      upload: {
+        skipSafeFetch: true,
+        staticDir: path.resolve(dirname, './media'),
+      },
+    },
     {
       slug: animatedTypeMedia,
       fields: [],

test/uploads/int.spec.ts

@@ -1,4 +1,4 @@
-import type { Payload } from 'payload'
+import type { CollectionSlug, Payload } from 'payload'
 
 import fs from 'fs'
 import path from 'path'
@@ -19,6 +19,7 @@ import {
   mediaSlug,
   reduceSlug,
   relationSlug,
+  skipSafeFetchMediaSlug,
   unstoredMediaSlug,
   usersSlug,
 } from './shared.js'
@@ -585,6 +586,22 @@ describe('Collections - Uploads', () => {
           )
         },
       )
+      it('should fetch when skipSafeFetch is enabled', async () => {
+        await expect(
+          payload.create({
+            collection: skipSafeFetchMediaSlug as CollectionSlug,
+            data: {
+              filename: 'test.png',
+              url: 'http://127.0.0.1/file.png',
+            },
+          }),
+        ).rejects.toThrow(
+          expect.objectContaining({
+            name: 'FileRetrievalError',
+            message: expect.not.stringContaining('unsafe'),
+          }),
+        )
+      })
     })
   })
 

test/uploads/shared.ts

@@ -25,7 +25,7 @@ export const withoutMetadataSlug = 'without-meta-data'
 export const withOnlyJPEGMetadataSlug = 'with-only-jpeg-meta-data'
 export const customFileNameMediaSlug = 'custom-file-name-media'
 export const allowListMediaSlug = 'allow-list-media'
-
+export const skipSafeFetchMediaSlug = 'skip-safe-fetch-media'
 export const listViewPreviewSlug = 'list-view-preview'
 export const threeDimensionalSlug = 'three-dimensional'
 export const constructorOptionsSlug = 'constructor-options'