Skip to content

Commit ea502bf

Browse files
author
Matthias Koeppe
committed
Merge remote-tracking branch 'upstream/develop' into r-tests
2 parents db5bf6c + fc3a044 commit ea502bf

File tree

7,215 files changed

+283716
-178065
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

7,215 files changed

+283716
-178065
lines changed

‎.ci/create-changes-html.sh‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/sh
2+
if [ $# != 2 ]; then
3+
echo >&2 "usage: $0 BASE_DOC_COMMIT DOC_REPO"
4+
echo >&2 "creates CHANGES.html in the current directory"
5+
echo >&2 "for the diffs of DOC_REPO against BASE_DOC_COMMIT"
6+
exit 1
7+
fi
8+
BASE_DOC_COMMIT="$1"
9+
DOC_REPOSITORY="$2"
10+
11+
# Create CHANGES.html
12+
echo '<html>' > CHANGES.html
13+
echo '<head>' >> CHANGES.html
14+
echo '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css">' >> CHANGES.html
15+
echo '<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>' >> CHANGES.html
16+
echo '<script>hljs.highlightAll();</script>' >> CHANGES.html
17+
cat >> CHANGES.html << EOF
18+
<script>
19+
document.addEventListener('DOMContentLoaded', () => {
20+
// This URL is hardcoded in the file .github/workflows/doc-publish.yml.
21+
// See NETLIFY_ALIAS of the "Deploy to Netlify" step.
22+
const baseDocURL = 'https://doc-develop--sagemath.netlify.app'
23+
const diffSite = 'https://pianomister.github.io/diffsite'
24+
const diffParagraphs = document.querySelectorAll('p.diff');
25+
diffParagraphs.forEach(paragraph => {
26+
const rootURL = window.location.origin;
27+
const docAnchor = paragraph.querySelector('a'); // first "a" element
28+
const url = new URL(docAnchor.href);
29+
const path = url.pathname;
30+
const anchor = document.createElement('a');
31+
anchor.href = diffSite + '/?url1=' + rootURL + path + '&url2=' + baseDocURL + path;
32+
anchor.textContent = 'compare with the base';
33+
anchor.setAttribute('target', '_blank');
34+
paragraph.appendChild(anchor);
35+
paragraph.innerHTML += '&nbsp;';
36+
const hunkAnchors = paragraph.querySelectorAll('a.hunk');
37+
hunkAnchors.forEach(hunkAnchor => {
38+
const url = new URL(hunkAnchor.href);
39+
const path = url.pathname;
40+
const pathHash = path + url.hash.replace('#', '%23');
41+
const anchor = document.createElement('a');
42+
anchor.href = diffSite + '/?url1=' + rootURL + pathHash + '&url2=' + baseDocURL + path;
43+
anchor.textContent = hunkAnchor.textContent;
44+
anchor.setAttribute('target', '_blank');
45+
paragraph.appendChild(anchor);
46+
paragraph.innerHTML += '&nbsp;';
47+
});
48+
});
49+
});
50+
</script>
51+
EOF
52+
echo '</head>' >> CHANGES.html
53+
echo '<body>' >> CHANGES.html
54+
(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- "*.html") > diff.txt
55+
python3 - << EOF
56+
import os, re, html
57+
with open('diff.txt', 'r') as f:
58+
diff_text = f.read()
59+
diff_blocks = re.split(r'^(?=diff --git)', diff_text, flags=re.MULTILINE)
60+
out_blocks = []
61+
for block in diff_blocks:
62+
match = re.search(r'^diff --git a/(.*) b/\1', block, flags=re.MULTILINE)
63+
if match:
64+
doc = match.group(1)
65+
file_path = os.path.join('$DOC_REPOSITORY', doc)
66+
try:
67+
with open(file_path, 'r') as file:
68+
content = file.readlines()
69+
except FileNotFoundError:
70+
content = []
71+
count = 0
72+
for line in block.splitlines():
73+
if line.startswith('@@ -'):
74+
search_result = re.search(r'@@ -(\d+),(\d+) \+(\d+),(\d+)', line)
75+
if search_result:
76+
line_number = int(search_result.group(3))
77+
for i in range(line_number - 1, -1, -1):
78+
if content[i].startswith('<'):
79+
count += 1
80+
content[i] = f'<span id="hunk{count}" style="visibility: hidden;"></span>' + content[i]
81+
break
82+
if content:
83+
with open(file_path, 'w') as file:
84+
file.writelines(content)
85+
path = doc
86+
hunks = '&nbsp;'.join(f'<a href="{path}#hunk{i+1}" class="hunk" target="_blank">#{i + 1}</a>' for i in range(count))
87+
out_blocks.append(f'<p class="diff"><a href="{path}">{doc}</a>&nbsp;' + hunks + '&emsp;</p>'
88+
+ '\n<pre><code class="language-diff">'
89+
+ html.escape(block).strip() + '</code></pre>')
90+
output_text = '\n'.join(out_blocks)
91+
with open('diff.html', 'w') as f:
92+
f.write(output_text)
93+
EOF
94+
cat diff.html >> CHANGES.html
95+
echo '</body>' >> CHANGES.html
96+
echo '</html>' >> CHANGES.html
97+
rm diff.txt diff.html

‎.ci/describe-system.sh‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎.ci/docker-exec-script.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/sh -x
2+
if [ $# -lt 3 ]; then
3+
echo >&2 "usage: docker-exec-script.sh CONTAINER WORKDIR [VAR=VALUE...] SCRIPT"
4+
exit 1
5+
fi
6+
CONTAINER=$1
7+
WORKDIR=$2
8+
shift 2
9+
(echo "cd \"$WORKDIR\"";
10+
while [ $# -gt 1 ]; do
11+
echo "export \"$1\""
12+
shift
13+
done;
14+
cat "$1") | docker exec -i $CONTAINER bash -ex

‎.ci/merge-fixes.sh‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#!/bin/sh
2+
# Apply open PRs labeled "p: CI Fix" from sagemath/sage as patches.
3+
# (policy set by vote in 2024-03,
4+
# https://groups.google.com/g/sage-devel/c/OKwwUGyKveo/m/vpyCXYBqAAAJ)
5+
#
6+
# This script is invoked by various workflows in .github/workflows
7+
#
8+
# The repository variable SAGE_CI_FIXES_FROM_REPOS can be set
9+
# to customize this for CI runs in forks:
10+
#
11+
# - If set to a whitespace-separated list of repositories, use them instead of sagemath/sage.
12+
# - If set to "none", do not apply any PRs.
13+
#
14+
# https://docs.github.com/en/actions/learn-github-actions/variables#creating-configuration-variables-for-a-repository
15+
export GIT_AUTHOR_NAME="ci-sage workflow"
16+
export GIT_AUTHOR_EMAIL="[email protected]"
17+
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
18+
export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
19+
mkdir -p upstream
20+
for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
21+
case $REPO in
22+
none)
23+
echo "Nothing to do for 'none' in SAGE_CI_FIXES_FROM_REPOSITORIES"
24+
;;
25+
*/*)
26+
echo "Getting open PRs with 'p: CI Fix' label from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+CI+Fix%22"
27+
GH="gh -R $REPO"
28+
REPO_FILE="upstream/ci-fixes-${REPO%%/*}-${REPO##*/}"
29+
PRs="$($GH pr list --label "p: CI Fix" --json number --jq '.[].number' | tee $REPO_FILE)"
30+
date -u +"%Y-%m-%dT%H:%M:%SZ" > $REPO_FILE.date # Record the date, for future reference
31+
if [ -z "$PRs" ]; then
32+
echo "Nothing to do: Found no open PRs with 'p: CI Fix' label in $REPO."
33+
else
34+
echo "Found open PRs with 'p: CI Fix' label in $REPO: $(echo $PRs)"
35+
git tag -f test_base
36+
git commit -q -m "Uncommitted changes" --no-allow-empty -a
37+
for a in $PRs; do
38+
# We used to pull the branch and merge it (after unshallowing), but when run on PRs that are
39+
# based on older releases, it has the side effect of updating to this release,
40+
# which may be confusing.
41+
#
42+
# Here, we obtain the "git am"-able patch of the PR branch.
43+
# This also makes it unnecessary to unshallow the repository.
44+
#
45+
# Considered alternative: Use https://github.com/$REPO/pull/$a.diff,
46+
# which squashes everything into one diff without commit metadata.
47+
PULL_URL="https://github.com/$REPO/pull/$a"
48+
PULL_SHORT="$REPO#$a"
49+
PULL_FILE="$REPO_FILE-$a"
50+
PATH=build/bin:$PATH build/bin/sage-download-file --quiet "$PULL_URL.patch" $PULL_FILE.patch
51+
date -u +"%Y-%m-%dT%H:%M:%SZ" > $PULL_FILE.date # Record the date, for future reference
52+
LAST_SHA=$(sed -n -E '/^From [0-9a-f]{40}/s/^From ([0-9a-f]{40}).*/\1/p' $PULL_FILE.patch | tail -n 1)
53+
COMMITS_URL="https://github.com/$REPO/commits/$LAST_SHA"
54+
echo "::group::Applying PR $PULL_URL @ $COMMITS_URL as a patch"
55+
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME applying $PULL_URL @ $COMMITS_URL"
56+
if git am --signoff --empty=keep < $PULL_FILE.patch; then
57+
echo "---- Applied patch --------------------------------------------------------------------------------"
58+
cat $PULL_FILE.patch
59+
echo "--------------------------------------------------------------------8<-----------------------------"
60+
echo "::endgroup::"
61+
elif git am --abort \
62+
&& if git fetch --unshallow --all > /dev/null 2>&1; then echo "Unshallowed"; fi \
63+
&& echo "Retrying with 3-way merge" \
64+
&& git am --empty=keep --3way < $PULL_FILE.patch; then
65+
echo "---- Applied patch --------------------------------------------------------------------------------"
66+
cat $PULL_FILE.patch
67+
echo "--------------------------------------------------------------------8<-----------------------------"
68+
echo "::endgroup::"
69+
else
70+
echo "---- Failure applying patch -----------------------------------------------------------------------"
71+
git am --signoff --show-current-patch=diff
72+
echo "--------------------------------------------------------------------8<-----------------------------"
73+
echo "::endgroup::"
74+
echo "Failure applying $PULL_SHORT as a patch, resetting"
75+
git am --signoff --abort
76+
fi
77+
done
78+
fi
79+
;;
80+
*)
81+
echo "Repository variable SAGE_CI_FIXES_FROM_REPOSITORIES, if set, should be a whitespace-separated list of repositories or 'none'"
82+
echo "Got: $SAGE_CI_FIXES_FROM_REPOSITORIES"
83+
exit 1
84+
;;
85+
esac
86+
done

‎.ci/protect-secrets.sh‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎.ci/retrofit-worktree.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/sh
2+
if [ $# != 2 ]; then
3+
echo >&2 "usage: $0 WORKTREE_NAME WORKTREE_DIRECTORY"
4+
echo >&2 "Ensures that the current working directory is a git repository,"
5+
echo >&2 "then makes WORKTREE_DIRECTORY a git worktree named WORKTREE_NAME."
6+
fi
7+
WORKTREE_NAME="$1"
8+
WORKTREE_DIRECTORY="$2"
9+
10+
export GIT_AUTHOR_NAME="ci-sage workflow"
11+
export GIT_AUTHOR_EMAIL="[email protected]"
12+
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
13+
export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
14+
15+
set -e
16+
17+
# Set globally for other parts of the workflow
18+
git config --global user.name "$GIT_AUTHOR_NAME"
19+
git config --global user.email "$GIT_AUTHOR_EMAIL"
20+
21+
set -x
22+
23+
# If actions/checkout downloaded our source tree using the GitHub REST API
24+
# instead of with git (because do not have git installed in our image),
25+
# we first make the source tree a repo.
26+
if [ ! -d .git ]; then git init; fi
27+
28+
# Commit and tag this state of the source tree "new". This is what we want to build and test.
29+
git add -A && git commit --quiet --allow-empty -m "new"
30+
git tag -f new
31+
32+
# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
33+
# But $WORKTREE_DIRECTORY is not a git repository.
34+
# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
35+
# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
36+
# Then we update worktree and index with "git checkout new".
37+
# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
38+
if [ -L $WORKTREE_NAME ]; then
39+
rm -f $WORKTREE_NAME
40+
fi
41+
git worktree prune --verbose
42+
git worktree add --detach $WORKTREE_NAME
43+
rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
44+
rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
45+
if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
46+
(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git checkout new && git status)

‎.ci/set_labels_by_changes.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bash
2+
3+
# cd to the PR directory
4+
cd $PR_MOUNT_DIR
5+
PR_BASE_SHA=$(git merge-base $BASE_SHA $PR_HEAD_SHA)
6+
7+
echo "set_labels_by_changes.sh called with environment:"
8+
echo "BASE SHA: $PR_BASE_SHA"
9+
echo "HEAD SHA: $PR_HEAD_SHA"
10+
echo "SMALL THRESHOLD $SMALL_THRESHOLD"
11+
echo "MODERATE THERESHOLD: $MODERATE_THRESHOLD"
12+
echo "LARGE THRESHOLD: $LARGE_THRESHOLD"
13+
14+
# get all the changes made and changed files
15+
CHANGES=$(git diff --ignore-all-space $PR_BASE_SHA $PR_HEAD_SHA)
16+
17+
# ignore blank lines
18+
CHANGES=$(echo "$CHANGES" | grep -vE '^[\+\-]\s*$')
19+
20+
# ignore non necessary lines from git diff
21+
CHANGES=$(echo "$CHANGES" | grep -E '^[+\-]' | grep -vE '^\+\+\+|^\-\-\-')
22+
23+
# count total no of lines
24+
CHANGES=$(echo "$CHANGES" | wc -l)
25+
26+
echo "CHANGES MADE: $CHANGES"
27+
28+
AUTH_HEADER="Authorization: Bearer $GITHUB_TOKEN"
29+
30+
MINIMAL="v: minimal"
31+
SMALL="v: small"
32+
MODERATE="v: moderate"
33+
LARGE="v: large"
34+
35+
DELETE_LABELS=("$MINIMAL" "$SMALL" "$MODERATE" "$LARGE")
36+
37+
if [ "$CHANGES" -gt "$LARGE_THRESHOLD" ]; then
38+
SIZE_LABEL="$LARGE"
39+
elif [ "$CHANGES" -gt "$MODERATE_THRESHOLD" ]; then
40+
SIZE_LABEL="$MODERATE"
41+
elif [ "$CHANGES" -gt "$SMALL_THRESHOLD" ]; then
42+
SIZE_LABEL="$SMALL"
43+
else
44+
SIZE_LABEL="$MINIMAL"
45+
fi
46+
47+
DELETE_LABELS=("${DELETE_LABELS[@]//${SIZE_LABEL}/}")
48+
49+
# API for adding labels on the Pull Request
50+
API_URL="https://api.github.com/repos/$REPOSITORY/issues/$PR_NUMBER/labels"
51+
52+
echo "Adding label: ${SIZE_LABEL[@]}"
53+
for LABEL in "${SIZE_LABEL[@]}"; do
54+
curl -X POST \
55+
-H "$AUTH_HEADER" \
56+
-H "Accept: application/vnd.github+json" \
57+
-H "X-GitHub-Api-Version: 2022-11-28" \
58+
-d "{\"labels\":[\"$LABEL\"]}" \
59+
"$API_URL" >/dev/null
60+
done
61+
62+
echo "Deleting Labels:"
63+
64+
for DELETE_LABEL in "${DELETE_LABELS[@]}"; do
65+
ENCODED_LABEL=$(echo "$DELETE_LABEL" | sed 's/ /%20/g')
66+
curl -X DELETE \
67+
-H "Accept: application/vnd.github+json" \
68+
-H "$AUTH_HEADER" \
69+
-H "X-GitHub-Api-Version: 2022-11-28" \
70+
"$API_URL/$ENCODED_LABEL" >/dev/null
71+
done

0 commit comments

Comments
 (0)