diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 583bfe80c9b..3c59e0d77ed 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -141,11 +141,14 @@ def CACHED_CHANGED_FILE_LIST = "cached_changed_file_list" def ACTION_INFO = "action_info" @Field def IMAGE_KEY_TO_TAG = "image_key_to_tag" +@Field +def TARGET_BRANCH = "target_branch" def globalVars = [ (GITHUB_PR_API_URL): gitlabParamsFromBot.get('github_pr_api_url', null), (CACHED_CHANGED_FILE_LIST): null, (ACTION_INFO): gitlabParamsFromBot.get('action_info', null), (IMAGE_KEY_TO_TAG): [:], + (TARGET_BRANCH): gitlabParamsFromBot.get('target_branch', null), ] // If not running all test stages in the L0 pre-merge, we will not update the GitLab status at the end. @@ -300,31 +303,72 @@ def echoNodeAndGpuInfo(pipeline, stageName) } def setupPipelineEnvironment(pipeline, testFilter, globalVars) +{ + sh "env | sort" + updateGitlabCommitStatus name: "${BUILD_STATUS_NAME}", state: 'running' + echo "Using GitLab repo: ${LLM_REPO}." + sh "git config --global --add safe.directory \"*\"" + // NB: getContainerURIs reads files in ${LLM_ROOT}/jenkins/ + if (env.gitlabMergeRequestLastCommit) { + env.gitlabCommit = env.gitlabMergeRequestLastCommit + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + } else { + branch = env.gitlabBranch ? env.gitlabBranch : "main" + trtllm_utils.checkoutSource(LLM_REPO, branch, LLM_ROOT, true, true) + checkoutCommit = sh (script: "cd ${LLM_ROOT} && git rev-parse HEAD",returnStdout: true).trim() + env.gitlabCommit = checkoutCommit + } + echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." + echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." + testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) + testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) + testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) + getContainerURIs().each { k, v -> + globalVars[k] = v + } +} + +def mergeWaiveList(pipeline, globalVars) +{ + // Get current waive list + sh "git config --global --add safe.directory \"*\"" + sh "cp ${LLM_ROOT}/tests/integration/test_lists/waives.txt ./waives_CUR_${env.gitlabCommit}.txt" + sh "cp ${LLM_ROOT}/jenkins/scripts/mergeWaiveList.py ./" + + // Get TOT waive list + LLM_TOT_ROOT = "llm-tot" + targetBranch = env.gitlabTargetBranch ? env.gitlabTargetBranch : globalVars[TARGET_BRANCH] + echo "Target branch: ${targetBranch}" + trtllm_utils.checkoutSource(LLM_REPO, targetBranch, LLM_TOT_ROOT, true, true) + targetBranchTOTCommit = sh (script: "cd ${LLM_TOT_ROOT} && git rev-parse HEAD", returnStdout: true).trim() + echo "Target branch TOT commit: ${targetBranchTOTCommit}" + sh "cp ${LLM_TOT_ROOT}/tests/integration/test_lists/waives.txt ./waives_TOT_${targetBranchTOTCommit}.txt" + + // Get waive list diff in current MR + def diff = getMergeRequestOneFileChanges(pipeline, globalVars, "tests/integration/test_lists/waives.txt") + + // Merge waive lists + sh """ + python3 mergeWaiveList.py \ + --cur-waive-list=waives_CUR_${env.gitlabCommit}.txt \ + --latest-waive-list=waives_TOT_${targetBranchTOTCommit}.txt \ + --diff='${diff}' \ + --output-file=waives.txt + """ + trtllm_utils.uploadArtifacts("waives*.txt", "${UPLOAD_PATH}/waive_list/") + echo "New merged test waive list: https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/waive_list/waives.txt" +} + +def preparation(pipeline, testFilter, globalVars) { image = "urm.nvidia.com/docker/golang:1.22" setupPipelineSpec = createKubernetesPodConfig(image, "package") trtllm_utils.launchKubernetesPod(pipeline, setupPipelineSpec, "trt-llm", { - sh "env | sort" - updateGitlabCommitStatus name: "${BUILD_STATUS_NAME}", state: 'running' - echo "Using GitLab repo: ${LLM_REPO}." - sh "git config --global --add safe.directory \"*\"" - // NB: getContainerURIs reads files in ${LLM_ROOT}/jenkins/ - if (env.gitlabMergeRequestLastCommit) { - env.gitlabCommit = env.gitlabMergeRequestLastCommit - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) - } else { - branch = env.gitlabBranch ? env.gitlabBranch : "main" - trtllm_utils.checkoutSource(LLM_REPO, branch, LLM_ROOT, true, true) - checkoutCommit = sh (script: "cd ${LLM_ROOT} && git rev-parse HEAD",returnStdout: true).trim() - env.gitlabCommit = checkoutCommit + stage("Setup Environment") { + setupPipelineEnvironment(pipeline, testFilter, globalVars) } - echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." - echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." - testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) - testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) - testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) - getContainerURIs().each { k, v -> - globalVars[k] = v + stage("Merge Test Waive List") { + mergeWaiveList(pipeline, globalVars) } }) } @@ -415,8 +459,8 @@ def launchReleaseCheck(pipeline) }) } -def getMergeRequestChangedFileListGitlab(pipeline) { - def changedFileList = [] +def getGitlabMRChangedFile(pipeline, function, filePath="") { + def result = null def pageId = 0 withCredentials([ usernamePassword( @@ -436,19 +480,34 @@ def getMergeRequestChangedFileListGitlab(pipeline) { returnStdout: true ) def rawDataList = readJSON text: rawDataJson, returnPojo: true - rawDataList.each { rawData -> - changedFileList += [rawData.get("old_path"), rawData.get("new_path")] + if (function == "getOneFileChanges") { + if (result == null) { + result = "" + } + rawDataList.find { rawData -> + if (rawData.get("new_path") == filePath || rawData.get("old_path") == filePath) { + result = rawData.get("diff") + return true + } + return false + } + if (result != "") { break } + } else if (function == "getChangedFileList") { + if (result == null) { + result = [] + } + rawDataList.each { rawData -> + result += [rawData.get("old_path"), rawData.get("new_path")] + } } if (!rawDataList) { break } } } - def changedFileListStr = changedFileList.join(",\n") - pipeline.echo("The changeset of this MR is: ${changedFileListStr}.") - return changedFileList + return result } -def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { - def changedFileList = [] +def getGithubMRChangedFile(pipeline, githubPrApiUrl, function, filePath="") { + def result = null def pageId = 0 withCredentials([ string( @@ -467,15 +526,30 @@ def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { ) echo "rawDataJson: ${rawDataJson}" def rawDataList = readJSON text: rawDataJson, returnPojo: true - rawDataList.each { rawData -> - changedFileList += [rawData.get("filename"), rawData.get("previous_filename")].findAll { it } + if (function == "getOneFileChanges") { + if (result == null) { + result = "" + } + rawDataList.find { rawData -> + if (rawData.get("filename") == filePath || rawData.get("previous_filename") == filePath) { + result = rawData.get("patch") + return true + } + return false + } + if (result != "") { break } + } else if (function == "getChangedFileList") { + if (result == null) { + result = [] + } + rawDataList.each { rawData -> + result += [rawData.get("filename"), rawData.get("previous_filename")].findAll { it } + } } if (!rawDataList) { break } } } - def changedFileListStr = changedFileList.join(",\n") - pipeline.echo("The changeset of this PR is: ${changedFileListStr}.") - return changedFileList + return result } def getMergeRequestChangedFileList(pipeline, globalVars) { @@ -491,11 +565,15 @@ def getMergeRequestChangedFileList(pipeline, globalVars) { return globalVars[CACHED_CHANGED_FILE_LIST] } try { + def changedFileList = [] if (githubPrApiUrl != null) { - globalVars[CACHED_CHANGED_FILE_LIST] = getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) + changedFileList = getGithubMRChangedFile(pipeline, githubPrApiUrl, "getChangedFileList") } else { - globalVars[CACHED_CHANGED_FILE_LIST] = getMergeRequestChangedFileListGitlab(pipeline) + changedFileList = getGitlabMRChangedFile(pipeline, "getChangedFileList") } + def changedFileListStr = changedFileList.join(",\n") + pipeline.echo("The changeset of this MR is: ${changedFileListStr}.") + globalVars[CACHED_CHANGED_FILE_LIST] = changedFileList return globalVars[CACHED_CHANGED_FILE_LIST] } catch (InterruptedException e) { throw e @@ -506,6 +584,26 @@ def getMergeRequestChangedFileList(pipeline, globalVars) { } } +def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) { + def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] + def diff = "" + + try { + if (githubPrApiUrl != null) { + diff = getGithubMRChangedFile(pipeline, githubPrApiUrl, "getOneFileChanges", filePath) + } else { + diff = getGitlabMRChangedFile(pipeline, "getOneFileChanges", filePath) + } + pipeline.echo("The change of ${filePath} is: ${diff}") + return diff + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + pipeline.echo("Get merge request one changed file diff failed. Error: ${e.toString()}") + return "" + } +} + def getAutoTriggerTagList(pipeline, testFilter, globalVars) { def autoTriggerTagList = [] def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) @@ -1136,12 +1234,12 @@ pipeline { } } stages { - stage("Setup environment") + stage("Preparation") { steps { script { - setupPipelineEnvironment(this, testFilter, globalVars) + preparation(this, testFilter, globalVars) println globalVars globalVars[ACTION_INFO] = trtllm_utils.setupPipelineDescription(this, globalVars[ACTION_INFO]) echo "enableFailFast is: ${enableFailFast}" diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 47326f5012f..ffa1fa3267c 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1290,6 +1290,22 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && wget -nv ${llmTarfile}") sh "cd ${llmPath} && tar -zxf ${tarName}" + // Download the new merged waives.txt + def waivesTxt = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/waive_list/waives.txt" + try { + trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${waivesTxt}") + if (!fileExists("waives.txt")) { + error "There is no merged waives.txt file, use the default waives.txt." + } + sh "rm ${llmSrc}/tests/integration/test_lists/waives.txt" + sh "mv waives.txt ${llmSrc}/tests/integration/test_lists/waives.txt" + echo "Download merged waives.txt successfully" + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + echo "Failed to download merged waives.txt, use the default waives.txt. Error: ${e.message}" + } + // install python package if (env.alternativeTRT) { sh "cd ${llmSrc} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" diff --git a/jenkins/scripts/mergeWaiveList.py b/jenkins/scripts/mergeWaiveList.py new file mode 100644 index 00000000000..418a603b9af --- /dev/null +++ b/jenkins/scripts/mergeWaiveList.py @@ -0,0 +1,56 @@ +import argparse +import sys + +# Generate the merged waive list: +# 1. Parse the current MR waive list, and get the removed lines from the diff +# 2. Parse the TOT waive list +# 3. Merge the current MR waive list and TOT waive list, and remove the removed lines from the step 1 + + +def get_remove_lines_from_diff(diff): + lines = diff.split('\n') + remove_lines = [ + line[1:] + '\n' for line in lines + if len(line) > 1 and line.startswith('-') + ] + return remove_lines + + +def parse_waive_txt(waive_txt): + with open(waive_txt, 'r') as f: + lines = f.readlines() + waive_list = [line for line in lines if line.strip()] + return waive_list + + +def write_waive_list(waive_list, output_file): + with open(output_file, 'w') as f: + for line in waive_list: + f.write(line) + + +def merge_waive_list(cur_list, main_list, remove_lines, output_file): + merged = list(dict.fromkeys(cur_list + main_list)) + for line in reversed(remove_lines): + for i in range(len(merged) - 1, -1, -1): + if merged[i] == line: + merged.pop(i) + break + write_waive_list(merged, output_file) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--cur-waive-list', + required=True, + help='Current waive list') + parser.add_argument('--latest-waive-list', + required=True, + help='Latest waive list') + parser.add_argument('--diff', required=True, help='Diff of the waive list') + parser.add_argument('--output-file', required=True, help='Output file') + args = parser.parse_args(sys.argv[1:]) + cur_list = parse_waive_txt(args.cur_waive_list) + main_list = parse_waive_txt(args.latest_waive_list) + remove_lines = get_remove_lines_from_diff(args.diff) + merge_waive_list(cur_list, main_list, remove_lines, args.output_file)