Skip to content

Commit fc80382

Browse files
slfan1989slfan1989
authored
Merge branch 'trunk' into YARN-11258 
2 parents 2baf6b2 + e614453 commit fc80382

File tree

252 files changed

+19625
-2542
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

252 files changed

+19625
-2542
lines changed

‎LICENSE-binary‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -258,36 +258,36 @@ io.grpc:grpc-netty:1.69.0
258258
io.grpc:grpc-protobuf:1.69.0
259259
io.grpc:grpc-protobuf-lite:1.69.0
260260
io.grpc:grpc-stub:1.69.0
261-
io.netty:netty-all:4.1.116.Final
262-
io.netty:netty-buffer:4.1.116.Final
263-
io.netty:netty-codec:4.1.116.Final
264-
io.netty:netty-codec-dns:4.1.116.Final
265-
io.netty:netty-codec-haproxy:4.1.116.Final
266-
io.netty:netty-codec-http:4.1.116.Final
267-
io.netty:netty-codec-http2:4.1.116.Final
268-
io.netty:netty-codec-memcache:4.1.116.Final
269-
io.netty:netty-codec-mqtt:4.1.116.Final
270-
io.netty:netty-codec-redis:4.1.116.Final
271-
io.netty:netty-codec-smtp:4.1.116.Final
272-
io.netty:netty-codec-socks:4.1.116.Final
273-
io.netty:netty-codec-stomp:4.1.116.Final
274-
io.netty:netty-codec-xml:4.1.116.Final
275-
io.netty:netty-common:4.1.116.Final
276-
io.netty:netty-handler:4.1.116.Final
277-
io.netty:netty-handler-proxy:4.1.116.Final
278-
io.netty:netty-resolver:4.1.116.Final
279-
io.netty:netty-resolver-dns:4.1.116.Final
280-
io.netty:netty-transport:4.1.116.Final
281-
io.netty:netty-transport-rxtx:4.1.116.Final
282-
io.netty:netty-transport-sctp:4.1.116.Final
283-
io.netty:netty-transport-udt:4.1.116.Final
284-
io.netty:netty-transport-classes-epoll:4.1.116.Final
285-
io.netty:netty-transport-native-unix-common:4.1.116.Final
286-
io.netty:netty-transport-classes-kqueue:4.1.116.Final
287-
io.netty:netty-resolver-dns-classes-macos:4.1.116.Final
288-
io.netty:netty-transport-native-epoll:4.1.116.Final
289-
io.netty:netty-transport-native-kqueue:4.1.116.Final
290-
io.netty:netty-resolver-dns-native-macos:4.1.116.Final
261+
io.netty:netty-all:4.1.118.Final
262+
io.netty:netty-buffer:4.1.118.Final
263+
io.netty:netty-codec:4.1.118.Final
264+
io.netty:netty-codec-dns:4.1.118.Final
265+
io.netty:netty-codec-haproxy:4.1.118.Final
266+
io.netty:netty-codec-http:4.1.118.Final
267+
io.netty:netty-codec-http2:4.1.118.Final
268+
io.netty:netty-codec-memcache:4.1.118.Final
269+
io.netty:netty-codec-mqtt:4.1.118.Final
270+
io.netty:netty-codec-redis:4.1.118.Final
271+
io.netty:netty-codec-smtp:4.1.118.Final
272+
io.netty:netty-codec-socks:4.1.118.Final
273+
io.netty:netty-codec-stomp:4.1.118.Final
274+
io.netty:netty-codec-xml:4.1.118.Final
275+
io.netty:netty-common:4.1.118.Final
276+
io.netty:netty-handler:4.1.118.Final
277+
io.netty:netty-handler-proxy:4.1.118.Final
278+
io.netty:netty-resolver:4.1.118.Final
279+
io.netty:netty-resolver-dns:4.1.118.Final
280+
io.netty:netty-transport:4.1.118.Final
281+
io.netty:netty-transport-rxtx:4.1.118.Final
282+
io.netty:netty-transport-sctp:4.1.118.Final
283+
io.netty:netty-transport-udt:4.1.118.Final
284+
io.netty:netty-transport-classes-epoll:4.1.118.Final
285+
io.netty:netty-transport-native-unix-common:4.1.118.Final
286+
io.netty:netty-transport-classes-kqueue:4.1.118.Final
287+
io.netty:netty-resolver-dns-classes-macos:4.1.118.Final
288+
io.netty:netty-transport-native-epoll:4.1.118.Final
289+
io.netty:netty-transport-native-kqueue:4.1.118.Final
290+
io.netty:netty-resolver-dns-native-macos:4.1.118.Final
291291
io.opencensus:opencensus-api:0.12.3
292292
io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
293293
io.reactivex:rxjava:1.3.8
@@ -318,7 +318,7 @@ org.apache.htrace:htrace-core:3.1.0-incubating
318318
org.apache.htrace:htrace-core4:4.1.0-incubating
319319
org.apache.httpcomponents:httpclient:4.5.13
320320
org.apache.httpcomponents:httpcore:4.4.13
321-
org.apache.kafka:kafka-clients:3.4.0
321+
org.apache.kafka:kafka-clients:3.9.0
322322
org.apache.kerby:kerb-admin:2.0.3
323323
org.apache.kerby:kerb-client:2.0.3
324324
org.apache.kerby:kerb-common:2.0.3
@@ -378,7 +378,7 @@ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/com
378378
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
379379
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
380380

381-
com.github.luben:zstd-jni:1.5.2-1
381+
com.github.luben:zstd-jni:1.5.6-4
382382
dnsjava:dnsjava:3.6.1
383383
org.codehaus.woodstox:stax2-api:4.2.1
384384

‎dev-support/docker/Dockerfile_windows_10‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,14 @@ RUN powershell git clone https://github.com/microsoft/vcpkg.git \
5151
&& cd vcpkg \
5252
&& git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d \
5353
&& .\bootstrap-vcpkg.bat
54+
55+
# Fix wrong download link for 7-zip in .\vcpkg\scripts\vcpkgTools.xml:
56+
# - https://www.nuget.org/api/v2/package/7-Zip.CommandLine/18.1.0 (not working anymore) is replaced with
57+
# - https://globalcdn.nuget.org/packages/7-zip.commandline.18.1.0.nupkg?packageVersion=18.1.0 (working)
58+
# Replacing is done using the Linux tool "sed".
59+
RUN choco install sed -y
60+
RUN sed -i "s,https://www.nuget.org/api/v2/package/7-Zip.CommandLine/18.1.0,https://globalcdn.nuget.org/packages/7-zip.commandline.18.1.0.nupkg?packageVersion=18.1.0,g" .\vcpkg\scripts\vcpkgTools.xml
61+
5462
RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
5563
RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
5664
RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
@@ -74,10 +82,10 @@ RUN powershell Expand-Archive -Path $Env:TEMP\zstd-v1.5.4-win64.zip -Destination
7482
RUN setx PATH "%PATH%;C:\ZStd"
7583

7684
# Install libopenssl 3.1.0 needed for rsync 3.2.7.
77-
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.0-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst
78-
RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar
85+
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.0-2-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.0-2-x86_64.pkg.tar.zst
86+
RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-2-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.0-2-x86_64.pkg.tar
7987
RUN powershell mkdir "C:\LibOpenSSL"
80-
RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C "C:\LibOpenSSL"
88+
RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-2-x86_64.pkg.tar -C "C:\LibOpenSSL"
8189

8290
# Install libxxhash 0.8.1 needed for rsync 3.2.7.
8391
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst
@@ -86,10 +94,10 @@ RUN powershell mkdir "C:\LibXXHash"
8694
RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash"
8795

8896
# Install libzstd 1.5.4 needed for rsync 3.2.7.
89-
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst
90-
RUN powershell zstd -d $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst -o $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar
97+
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.5-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.5-1-x86_64.pkg.tar.zst
98+
RUN powershell zstd -d $Env:TEMP\libzstd-1.5.5-1-x86_64.pkg.tar.zst -o $Env:TEMP\libzstd-1.5.5-1-x86_64.pkg.tar
9199
RUN powershell mkdir "C:\LibZStd"
92-
RUN powershell tar -xvf $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar -C "C:\LibZStd"
100+
RUN powershell tar -xvf $Env:TEMP\libzstd-1.5.5-1-x86_64.pkg.tar -C "C:\LibZStd"
93101

94102
# Install rsync 3.2.7.
95103
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/rsync-3.2.7-2-x86_64.pkg.tar.zst -OutFile $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import java.text.SimpleDateFormat;
2323
import java.util.Date;
2424
import java.util.LinkedList;
25-
import java.util.TimeZone;
2625

2726
import org.apache.hadoop.classification.InterfaceAudience;
2827
import org.apache.hadoop.classification.InterfaceStability;
@@ -73,7 +72,6 @@ public static void registerCommands(CommandFactory factory) {
7372
protected final SimpleDateFormat timeFmt;
7473
{
7574
timeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
76-
timeFmt.setTimeZone(TimeZone.getTimeZone("UTC"));
7775
}
7876

7977
// default format string

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java‎

Lines changed: 82 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,8 @@ public class Client implements AutoCloseable {
9696
private static final ThreadLocal<Integer> retryCount = new ThreadLocal<Integer>();
9797
private static final ThreadLocal<Object> EXTERNAL_CALL_HANDLER
9898
= new ThreadLocal<>();
99-
private static final ThreadLocal<AsyncGet<? extends Writable, IOException>>
100-
ASYNC_RPC_RESPONSE = new ThreadLocal<>();
99+
private static final ThreadLocal<CompletableFuture<Writable>> ASYNC_RPC_RESPONSE
100+
= new ThreadLocal<>();
101101
private static final ThreadLocal<Boolean> asynchronousMode =
102102
new ThreadLocal<Boolean>() {
103103
@Override
@@ -110,7 +110,46 @@ protected Boolean initialValue() {
110110
@Unstable
111111
public static <T extends Writable> AsyncGet<T, IOException>
112112
getAsyncRpcResponse() {
113-
return (AsyncGet<T, IOException>) ASYNC_RPC_RESPONSE.get();
113+
CompletableFuture<Writable> responseFuture = ASYNC_RPC_RESPONSE.get();
114+
return new AsyncGet<T, IOException>() {
115+
@Override
116+
public T get(long timeout, TimeUnit unit)
117+
throws IOException, TimeoutException, InterruptedException {
118+
try {
119+
if (unit == null || timeout < 0) {
120+
return (T) responseFuture.get();
121+
}
122+
return (T) responseFuture.get(timeout, unit);
123+
} catch (ExecutionException e) {
124+
Throwable cause = e.getCause();
125+
if (cause instanceof IOException) {
126+
throw (IOException) cause;
127+
}
128+
throw new IllegalStateException(e);
129+
}
130+
}
131+
132+
@Override
133+
public boolean isDone() {
134+
return responseFuture.isDone();
135+
}
136+
};
137+
}
138+
139+
/**
140+
* Retrieves the current response future from the thread-local storage.
141+
*
142+
* @return A {@link CompletableFuture} of type T that represents the
143+
* asynchronous operation. If no response future is present in
144+
* the thread-local storage, this method returns {@code null}.
145+
* @param <T> The type of the value completed by the returned
146+
* {@link CompletableFuture}. It must be a subclass of
147+
* {@link Writable}.
148+
* @see CompletableFuture
149+
* @see Writable
150+
*/
151+
public static <T extends Writable> CompletableFuture<T> getResponseFuture() {
152+
return (CompletableFuture<T>) ASYNC_RPC_RESPONSE.get();
114153
}
115154

116155
/**
@@ -277,10 +316,8 @@ static class Call {
277316
final int id; // call id
278317
final int retry; // retry count
279318
final Writable rpcRequest; // the serialized rpc request
280-
Writable rpcResponse; // null if rpc has error
281-
IOException error; // exception, null if success
319+
private final CompletableFuture<Writable> rpcResponseFuture;
282320
final RPC.RpcKind rpcKind; // Rpc EngineKind
283-
boolean done; // true when call is done
284321
private final Object externalHandler;
285322
private AlignmentContext alignmentContext;
286323

@@ -304,6 +341,7 @@ private Call(RPC.RpcKind rpcKind, Writable param) {
304341
}
305342

306343
this.externalHandler = EXTERNAL_CALL_HANDLER.get();
344+
this.rpcResponseFuture = new CompletableFuture<>();
307345
}
308346

309347
@Override
@@ -314,9 +352,6 @@ public String toString() {
314352
/** Indicate when the call is complete and the
315353
* value or error are available. Notifies by default. */
316354
protected synchronized void callComplete() {
317-
this.done = true;
318-
notify(); // notify caller
319-
320355
if (externalHandler != null) {
321356
synchronized (externalHandler) {
322357
externalHandler.notify();
@@ -339,7 +374,7 @@ public synchronized void setAlignmentContext(AlignmentContext ac) {
339374
* @param error exception thrown by the call; either local or remote
340375
*/
341376
public synchronized void setException(IOException error) {
342-
this.error = error;
377+
rpcResponseFuture.completeExceptionally(error);
343378
callComplete();
344379
}
345380

@@ -349,13 +384,9 @@ public synchronized void setException(IOException error) {
349384
* @param rpcResponse return value of the rpc call.
350385
*/
351386
public synchronized void setRpcResponse(Writable rpcResponse) {
352-
this.rpcResponse = rpcResponse;
387+
rpcResponseFuture.complete(rpcResponse);
353388
callComplete();
354389
}
355-
356-
public synchronized Writable getRpcResponse() {
357-
return rpcResponse;
358-
}
359390
}
360391

361392
/** Thread that reads responses and notifies callers. Each connection owns a
@@ -1495,39 +1526,19 @@ Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
14951526
}
14961527

14971528
if (isAsynchronousMode()) {
1498-
final AsyncGet<Writable, IOException> asyncGet
1499-
= new AsyncGet<Writable, IOException>() {
1500-
@Override
1501-
public Writable get(long timeout, TimeUnit unit)
1502-
throws IOException, TimeoutException{
1503-
boolean done = true;
1504-
try {
1505-
final Writable w = getRpcResponse(call, connection, timeout, unit);
1506-
if (w == null) {
1507-
done = false;
1508-
throw new TimeoutException(call + " timed out "
1509-
+ timeout + " " + unit);
1510-
}
1511-
return w;
1512-
} finally {
1513-
if (done) {
1514-
releaseAsyncCall();
1529+
CompletableFuture<Writable> result = call.rpcResponseFuture.handle(
1530+
(rpcResponse, e) -> {
1531+
releaseAsyncCall();
1532+
if (e != null) {
1533+
IOException ioe = (IOException) e;
1534+
throw new CompletionException(warpIOException(ioe, connection));
15151535
}
1516-
}
1517-
}
1518-
1519-
@Override
1520-
public boolean isDone() {
1521-
synchronized (call) {
1522-
return call.done;
1523-
}
1524-
}
1525-
};
1526-
1527-
ASYNC_RPC_RESPONSE.set(asyncGet);
1536+
return rpcResponse;
1537+
});
1538+
ASYNC_RPC_RESPONSE.set(result);
15281539
return null;
15291540
} else {
1530-
return getRpcResponse(call, connection, -1, null);
1541+
return getRpcResponse(call, connection);
15311542
}
15321543
}
15331544

@@ -1564,37 +1575,34 @@ int getAsyncCallCount() {
15641575
}
15651576

15661577
/** @return the rpc response or, in case of timeout, null. */
1567-
private Writable getRpcResponse(final Call call, final Connection connection,
1568-
final long timeout, final TimeUnit unit) throws IOException {
1569-
synchronized (call) {
1570-
while (!call.done) {
1571-
try {
1572-
AsyncGet.Util.wait(call, timeout, unit);
1573-
if (timeout >= 0 && !call.done) {
1574-
return null;
1575-
}
1576-
} catch (InterruptedException ie) {
1577-
Thread.currentThread().interrupt();
1578-
throw new InterruptedIOException("Call interrupted");
1579-
}
1578+
private Writable getRpcResponse(final Call call, final Connection connection)
1579+
throws IOException {
1580+
try {
1581+
return call.rpcResponseFuture.get();
1582+
} catch (InterruptedException ie) {
1583+
Thread.currentThread().interrupt();
1584+
throw new InterruptedIOException("Call interrupted");
1585+
} catch (ExecutionException e) {
1586+
Throwable cause = e.getCause();
1587+
if (cause instanceof IOException) {
1588+
throw warpIOException((IOException) cause, connection);
15801589
}
1590+
throw new IllegalStateException(e);
1591+
}
1592+
}
15811593

1582-
if (call.error != null) {
1583-
if (call.error instanceof RemoteException ||
1584-
call.error instanceof SaslException) {
1585-
call.error.fillInStackTrace();
1586-
throw call.error;
1587-
} else { // local exception
1588-
InetSocketAddress address = connection.getRemoteAddress();
1589-
throw NetUtils.wrapException(address.getHostName(),
1590-
address.getPort(),
1591-
NetUtils.getHostname(),
1592-
0,
1593-
call.error);
1594-
}
1595-
} else {
1596-
return call.getRpcResponse();
1597-
}
1594+
private IOException warpIOException(IOException ioe, Connection connection) {
1595+
if (ioe instanceof RemoteException ||
1596+
ioe instanceof SaslException) {
1597+
ioe.fillInStackTrace();
1598+
return ioe;
1599+
} else { // local exception
1600+
InetSocketAddress address = connection.getRemoteAddress();
1601+
return NetUtils.wrapException(address.getHostName(),
1602+
address.getPort(),
1603+
NetUtils.getHostname(),
1604+
0,
1605+
ioe);
15981606
}
15991607
}
16001608

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ public Message invoke(Object proxy, final Method method, Object[] args)
214214
throws ServiceException {
215215
long startTime = 0;
216216
if (LOG.isDebugEnabled()) {
217-
startTime = Time.now();
217+
startTime = Time.monotonicNow();
218218
}
219219

220220
if (args.length != 2) { // RpcController + Message
@@ -267,7 +267,7 @@ public Message invoke(Object proxy, final Method method, Object[] args)
267267
}
268268

269269
if (LOG.isDebugEnabled()) {
270-
long callTime = Time.now() - startTime;
270+
long callTime = Time.monotonicNow() - startTime;
271271
LOG.debug("Call: " + method.getName() + " took " + callTime + "ms");
272272
}
273273

0 commit comments

Comments
 (0)