@@ -275,6 +275,40 @@ static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
275275 return min (pages , (sector_t )BIO_MAX_PAGES );
276276}
277277
278+ static int __blkdev_issue_zero_pages (struct block_device * bdev ,
279+ sector_t sector , sector_t nr_sects , gfp_t gfp_mask ,
280+ struct bio * * biop )
281+ {
282+ struct request_queue * q = bdev_get_queue (bdev );
283+ struct bio * bio = * biop ;
284+ int bi_size = 0 ;
285+ unsigned int sz ;
286+
287+ if (!q )
288+ return - ENXIO ;
289+
290+ while (nr_sects != 0 ) {
291+ bio = next_bio (bio , __blkdev_sectors_to_bio_pages (nr_sects ),
292+ gfp_mask );
293+ bio -> bi_iter .bi_sector = sector ;
294+ bio_set_dev (bio , bdev );
295+ bio_set_op_attrs (bio , REQ_OP_WRITE , 0 );
296+
297+ while (nr_sects != 0 ) {
298+ sz = min ((sector_t ) PAGE_SIZE , nr_sects << 9 );
299+ bi_size = bio_add_page (bio , ZERO_PAGE (0 ), sz , 0 );
300+ nr_sects -= bi_size >> 9 ;
301+ sector += bi_size >> 9 ;
302+ if (bi_size < sz )
303+ break ;
304+ }
305+ cond_resched ();
306+ }
307+
308+ * biop = bio ;
309+ return 0 ;
310+ }
311+
278312/**
279313 * __blkdev_issue_zeroout - generate number of zero filed write bios
280314 * @bdev: blockdev to issue
@@ -288,12 +322,6 @@ static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
288322 * Zero-fill a block range, either using hardware offload or by explicitly
289323 * writing zeroes to the device.
290324 *
291- * Note that this function may fail with -EOPNOTSUPP if the driver signals
292- * zeroing offload support, but the device fails to process the command (for
293- * some devices there is no non-destructive way to verify whether this
294- * operation is actually supported). In this case the caller should call
295- * retry the call to blkdev_issue_zeroout() and the fallback path will be used.
296- *
297325 * If a device is using logical block provisioning, the underlying space will
298326 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
299327 *
@@ -305,9 +333,6 @@ int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
305333 unsigned flags )
306334{
307335 int ret ;
308- int bi_size = 0 ;
309- struct bio * bio = * biop ;
310- unsigned int sz ;
311336 sector_t bs_mask ;
312337
313338 bs_mask = (bdev_logical_block_size (bdev ) >> 9 ) - 1 ;
@@ -317,30 +342,10 @@ int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
317342 ret = __blkdev_issue_write_zeroes (bdev , sector , nr_sects , gfp_mask ,
318343 biop , flags );
319344 if (ret != - EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK ))
320- goto out ;
321-
322- ret = 0 ;
323- while (nr_sects != 0 ) {
324- bio = next_bio (bio , __blkdev_sectors_to_bio_pages (nr_sects ),
325- gfp_mask );
326- bio -> bi_iter .bi_sector = sector ;
327- bio_set_dev (bio , bdev );
328- bio_set_op_attrs (bio , REQ_OP_WRITE , 0 );
329-
330- while (nr_sects != 0 ) {
331- sz = min ((sector_t ) PAGE_SIZE , nr_sects << 9 );
332- bi_size = bio_add_page (bio , ZERO_PAGE (0 ), sz , 0 );
333- nr_sects -= bi_size >> 9 ;
334- sector += bi_size >> 9 ;
335- if (bi_size < sz )
336- break ;
337- }
338- cond_resched ();
339- }
345+ return ret ;
340346
341- * biop = bio ;
342- out :
343- return ret ;
347+ return __blkdev_issue_zero_pages (bdev , sector , nr_sects , gfp_mask ,
348+ biop );
344349}
345350EXPORT_SYMBOL (__blkdev_issue_zeroout );
346351
@@ -360,18 +365,49 @@ EXPORT_SYMBOL(__blkdev_issue_zeroout);
360365int blkdev_issue_zeroout (struct block_device * bdev , sector_t sector ,
361366 sector_t nr_sects , gfp_t gfp_mask , unsigned flags )
362367{
363- int ret ;
364- struct bio * bio = NULL ;
368+ int ret = 0 ;
369+ sector_t bs_mask ;
370+ struct bio * bio ;
365371 struct blk_plug plug ;
372+ bool try_write_zeroes = !!bdev_write_zeroes_sectors (bdev );
366373
374+ bs_mask = (bdev_logical_block_size (bdev ) >> 9 ) - 1 ;
375+ if ((sector | nr_sects ) & bs_mask )
376+ return - EINVAL ;
377+
378+ retry :
379+ bio = NULL ;
367380 blk_start_plug (& plug );
368- ret = __blkdev_issue_zeroout (bdev , sector , nr_sects , gfp_mask ,
369- & bio , flags );
381+ if (try_write_zeroes ) {
382+ ret = __blkdev_issue_write_zeroes (bdev , sector , nr_sects ,
383+ gfp_mask , & bio , flags );
384+ } else if (!(flags & BLKDEV_ZERO_NOFALLBACK )) {
385+ ret = __blkdev_issue_zero_pages (bdev , sector , nr_sects ,
386+ gfp_mask , & bio );
387+ } else {
388+ /* No zeroing offload support */
389+ ret = - EOPNOTSUPP ;
390+ }
370391 if (ret == 0 && bio ) {
371392 ret = submit_bio_wait (bio );
372393 bio_put (bio );
373394 }
374395 blk_finish_plug (& plug );
396+ if (ret && try_write_zeroes ) {
397+ if (!(flags & BLKDEV_ZERO_NOFALLBACK )) {
398+ try_write_zeroes = false;
399+ goto retry ;
400+ }
401+ if (!bdev_write_zeroes_sectors (bdev )) {
402+ /*
403+ * Zeroing offload support was indicated, but the
404+ * device reported ILLEGAL REQUEST (for some devices
405+ * there is no non-destructive way to verify whether
406+ * WRITE ZEROES is actually supported).
407+ */
408+ ret = - EOPNOTSUPP ;
409+ }
410+ }
375411
376412 return ret ;
377413}
0 commit comments