From cb08136733b4b4dc48e488e33525dcebb715a75f Mon Sep 17 00:00:00 2001 From: baishuo Date: Wed, 16 Nov 2016 22:35:29 -0800 Subject: [PATCH] delete temporary folder after insert hive table --- .../apache/spark/sql/hive/execution/InsertIntoHiveTable.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala index eb0c31ced6586..b823b9494c8ef 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala @@ -292,6 +292,10 @@ case class InsertIntoHiveTable( holdDDLTime) } + // delete temporary folder after insert + val fileSystem = outputPath.getFileSystem(jobConf) + fileSystem.delete(outputPath.getParent, true) + // Invalidate the cache. sqlContext.sharedState.cacheManager.invalidateCache(table) sqlContext.sessionState.catalog.refreshTable(table.catalogTable.identifier)