Skip to content

Optimize git #98

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 25, 2024
Merged

Optimize git #98

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
20018e5
#95 Now using one instance of Git & one instance of Repository
jimbethancourt Sep 23, 2024
5db8af0
#95 Performing ScmLogInfo lookup in parallel
jimbethancourt Sep 25, 2024
ec3be73
Applying formatting
jimbethancourt Sep 25, 2024
7fed315
#91 Silenced PMD complaints about directories
jimbethancourt Sep 25, 2024
75449c5
#97 Cycle analysis included in Maven site report
jimbethancourt Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions change-proneness-ranker/src/main/java/org/hjug/git/ChangePronenessRanker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,17 @@
import java.util.*;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Repository;

@Slf4j
public class ChangePronenessRanker {

private final TreeMap<Integer, Integer> changeCountsByTimeStamps = new TreeMap<>();
private final Map<String, ScmLogInfo> cachedScmLogInfos = new HashMap<>();

public ChangePronenessRanker(Repository repository, GitLogReader repositoryLogReader) {
public ChangePronenessRanker(GitLogReader repositoryLogReader) {
try {
log.info("Capturing change count based on commit timestamps");
changeCountsByTimeStamps.putAll(repositoryLogReader.captureChangeCountByCommitTimestamp(repository));
changeCountsByTimeStamps.putAll(repositoryLogReader.captureChangeCountByCommitTimestamp());
} catch (IOException | GitAPIException e) {
log.error("Error reading from repository: {}", e.getMessage());
}
Expand Down
98 changes: 61 additions & 37 deletions change-proneness-ranker/src/main/java/org/hjug/git/GitLogReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,39 @@
import org.eclipse.jgit.util.io.NullOutputStream;

@Slf4j
public class GitLogReader {
public class GitLogReader implements AutoCloseable {

static final String JAVA_FILE_TYPE = ".java";

private Repository gitRepository;

private Git git;

public GitLogReader() {}

public GitLogReader(File basedir) throws IOException {
FileRepositoryBuilder repositoryBuilder = new FileRepositoryBuilder().findGitDir(basedir);
String gitIndexFileEnvVariable = System.getenv("GIT_INDEX_FILE");
if (Objects.nonNull(gitIndexFileEnvVariable)
&& !gitIndexFileEnvVariable.trim().isEmpty()) {
log.debug("Setting Index File based on Env Variable GIT_INDEX_FILE {}", gitIndexFileEnvVariable);
repositoryBuilder = repositoryBuilder.setIndexFile(new File(gitIndexFileEnvVariable));
}

git = Git.open(repositoryBuilder.getGitDir());
gitRepository = git.getRepository();
}

GitLogReader(Git git) {
this.git = git;
gitRepository = git.getRepository();
}

@Override
public void close() throws Exception {
git.close();
}

// Based on
// https://github.com/Cosium/git-code-format-maven-plugin/blob/master/src/main/java/com/cosium/code/format/AbstractMavenGitCodeFormatMojo.java
// MIT License
Expand Down Expand Up @@ -81,14 +110,12 @@ public Map<String, ByteArrayOutputStream> listRepositoryContentsAtHEAD(Repositor
* Returns the number of commits and earliest commit for a given path
* TODO: Move to a different class???
*
* @param repository
* @param path
* @return a LogInfo object
* @throws GitAPIException
*/
public ScmLogInfo fileLog(Repository repository, String path) throws GitAPIException, IOException {
Git git = new Git(repository);
ObjectId branchId = repository.resolve("HEAD");
public ScmLogInfo fileLog(String path) throws GitAPIException, IOException {
ObjectId branchId = gitRepository.resolve("HEAD");
Iterable<RevCommit> revCommits = git.log().add(branchId).addPath(path).call();

int commitCount = 0;
Expand All @@ -101,8 +128,7 @@ public ScmLogInfo fileLog(Repository repository, String path) throws GitAPIExcep
}

// based on https://stackoverflow.com/a/59274329/346247
int mostRecentCommit = new Git(repository)
.log()
int mostRecentCommit = git.log()
.add(branchId)
.addPath(path)
.setMaxCount(1)
Expand All @@ -115,49 +141,47 @@ public ScmLogInfo fileLog(Repository repository, String path) throws GitAPIExcep
}

// based on https://stackoverflow.com/questions/27361538/how-to-show-changes-between-commits-with-jgit
public TreeMap<Integer, Integer> captureChangeCountByCommitTimestamp(Repository repository)
throws IOException, GitAPIException {
public TreeMap<Integer, Integer> captureChangeCountByCommitTimestamp() throws IOException, GitAPIException {

TreeMap<Integer, Integer> changesByCommitTimestamp = new TreeMap<>();

try (Git git = new Git(repository)) {
ObjectId branchId = repository.resolve("HEAD");
Iterable<RevCommit> commits = git.log().add(branchId).call();
ObjectId branchId = gitRepository.resolve("HEAD");
Iterable<RevCommit> commits = git.log().add(branchId).call();

RevCommit newCommit = null;
RevCommit newCommit = null;

for (Iterator<RevCommit> iterator = commits.iterator(); iterator.hasNext(); ) {
RevCommit oldCommit = iterator.next();

int count = 0;
if (null == newCommit) {
newCommit = oldCommit;
continue;
}
for (Iterator<RevCommit> iterator = commits.iterator(); iterator.hasNext(); ) {
RevCommit oldCommit = iterator.next();

for (DiffEntry entry : getDiffEntries(git, newCommit, oldCommit)) {
if (entry.getNewPath().endsWith(JAVA_FILE_TYPE)
|| entry.getOldPath().endsWith(JAVA_FILE_TYPE)) {
count++;
}
}
int count = 0;
if (null == newCommit) {
newCommit = oldCommit;
continue;
}

if (count > 0) {
changesByCommitTimestamp.put(newCommit.getCommitTime(), count);
for (DiffEntry entry : getDiffEntries(newCommit, oldCommit)) {
if (entry.getNewPath().endsWith(JAVA_FILE_TYPE)
|| entry.getOldPath().endsWith(JAVA_FILE_TYPE)) {
count++;
}
}

// Handle first / initial commit
if (!iterator.hasNext()) {
changesByCommitTimestamp.putAll(walkFirstCommit(repository, oldCommit));
}
if (count > 0) {
changesByCommitTimestamp.put(newCommit.getCommitTime(), count);
}

newCommit = oldCommit;
// Handle first / initial commit
if (!iterator.hasNext()) {
changesByCommitTimestamp.putAll(walkFirstCommit(oldCommit));
}

newCommit = oldCommit;
}

return changesByCommitTimestamp;
}

private List<DiffEntry> getDiffEntries(Git git, RevCommit newCommit, RevCommit oldCommit) throws IOException {
private List<DiffEntry> getDiffEntries(RevCommit newCommit, RevCommit oldCommit) throws IOException {
CanonicalTreeParser oldTreeIter = new CanonicalTreeParser();
CanonicalTreeParser newTreeIter = new CanonicalTreeParser();
try (ObjectReader reader = git.getRepository().newObjectReader()) {
Expand All @@ -172,11 +196,11 @@ private List<DiffEntry> getDiffEntries(Git git, RevCommit newCommit, RevCommit o
return df.scan(oldTreeIter, newTreeIter);
}

Map<Integer, Integer> walkFirstCommit(Repository repository, RevCommit firstCommit) throws IOException {
Map<Integer, Integer> walkFirstCommit(RevCommit firstCommit) throws IOException {
Map<Integer, Integer> changesByCommitTimestamp = new TreeMap<>();
int firstCommitCount = 0;
ObjectId treeId = firstCommit.getTree();
try (TreeWalk treeWalk = new TreeWalk(repository)) {
try (TreeWalk treeWalk = new TreeWalk(gitRepository)) {
treeWalk.setRecursive(false);
treeWalk.reset(treeId);
while (treeWalk.next()) {
Expand Down
9 changes: 4 additions & 5 deletions change-proneness-ranker/src/test/java/org/hjug/git/ChangePronenessRankerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.hjug.git;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

Expand Down Expand Up @@ -31,9 +30,9 @@ void testChangePronenessCalculation() throws IOException, GitAPIException {
commitsWithChangeCounts.put(scmLogInfo.getEarliestCommit() + 5 * 60, 3);
commitsWithChangeCounts.put(scmLogInfo.getEarliestCommit() + 10 * 60, 3);

when(repositoryLogReader.captureChangeCountByCommitTimestamp(any())).thenReturn(commitsWithChangeCounts);
when(repositoryLogReader.captureChangeCountByCommitTimestamp()).thenReturn(commitsWithChangeCounts);

changePronenessRanker = new ChangePronenessRanker(null, repositoryLogReader);
changePronenessRanker = new ChangePronenessRanker(repositoryLogReader);
List<ScmLogInfo> scmLogInfos = new ArrayList<>();
scmLogInfos.add(scmLogInfo);
changePronenessRanker.rankChangeProneness(scmLogInfos);
Expand All @@ -57,8 +56,8 @@ void testRankChangeProneness() throws IOException, GitAPIException {
commitsWithChangeCounts.put(scmLogInfo2.getEarliestCommit() + 5 * 60, 5);
commitsWithChangeCounts.put(scmLogInfo2.getEarliestCommit() + 10 * 60, 5);

when(repositoryLogReader.captureChangeCountByCommitTimestamp(any())).thenReturn(commitsWithChangeCounts);
changePronenessRanker = new ChangePronenessRanker(null, repositoryLogReader);
when(repositoryLogReader.captureChangeCountByCommitTimestamp()).thenReturn(commitsWithChangeCounts);
changePronenessRanker = new ChangePronenessRanker(repositoryLogReader);

List<ScmLogInfo> scmLogInfos = new ArrayList<>();
scmLogInfos.add(scmLogInfo);
Expand Down
12 changes: 6 additions & 6 deletions change-proneness-ranker/src/test/java/org/hjug/git/GitLogReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ void testFileLog() throws IOException, GitAPIException, InterruptedException {
// This path works when referencing the full Tobago repository
// String filePath = "tobago-core/src/main/java/org/apache/myfaces/tobago/facelets/AttributeHandler.java";

GitLogReader gitLogReader = new GitLogReader();
GitLogReader gitLogReader = new GitLogReader(git);

String attributeHandler = "AttributeHandler.java";
InputStream resourceAsStream = getClass().getClassLoader().getResourceAsStream(attributeHandler);
Expand All @@ -59,7 +59,7 @@ void testFileLog() throws IOException, GitAPIException, InterruptedException {
git.add().addFilepattern(".").call();
RevCommit secondCommit = git.commit().setMessage("message").call();

ScmLogInfo scmLogInfo = gitLogReader.fileLog(repository, attributeHandler);
ScmLogInfo scmLogInfo = gitLogReader.fileLog(attributeHandler);

Assertions.assertEquals(2, scmLogInfo.getCommitCount());
Assertions.assertEquals(firstCommit.getCommitTime(), scmLogInfo.getEarliestCommit());
Expand All @@ -68,23 +68,23 @@ void testFileLog() throws IOException, GitAPIException, InterruptedException {

@Test
void testWalkFirstCommit() throws IOException, GitAPIException {
GitLogReader gitLogReader = new GitLogReader();
GitLogReader gitLogReader = new GitLogReader(git);

String attributeHandler = "AttributeHandler.java";
InputStream resourceAsStream = getClass().getClassLoader().getResourceAsStream(attributeHandler);
writeFile(attributeHandler, convertInputStreamToString(resourceAsStream));
git.add().addFilepattern(".").call();
RevCommit commit = git.commit().setMessage("message").call();

Map<Integer, Integer> result = gitLogReader.walkFirstCommit(repository, commit);
Map<Integer, Integer> result = gitLogReader.walkFirstCommit(commit);

Assertions.assertTrue(result.containsKey(commit.getCommitTime()));
Assertions.assertEquals(1, result.get(commit.getCommitTime()).intValue());
}

@Test
void testCaptureChangCountByCommitTimestamp() throws Exception {
GitLogReader gitLogReader = new GitLogReader();
GitLogReader gitLogReader = new GitLogReader(git);

String attributeHandler = "AttributeHandler.java";
InputStream resourceAsStream = getClass().getClassLoader().getResourceAsStream(attributeHandler);
Expand All @@ -106,7 +106,7 @@ void testCaptureChangCountByCommitTimestamp() throws Exception {
git.add().addFilepattern(".").call();
RevCommit secondCommit = git.commit().setMessage("message").call();

Map<Integer, Integer> commitCounts = gitLogReader.captureChangeCountByCommitTimestamp(repository);
Map<Integer, Integer> commitCounts = gitLogReader.captureChangeCountByCommitTimestamp();

Assertions.assertEquals(1, commitCounts.get(firstCommit.getCommitTime()).intValue());
Assertions.assertEquals(
Expand Down
61 changes: 32 additions & 29 deletions cost-benefit-calculator/src/main/java/org/hjug/cbc/CostBenefitCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import net.sourceforge.pmd.*;
import net.sourceforge.pmd.lang.LanguageRegistry;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Repository;
import org.hjug.cycledetector.CircularReferenceChecker;
import org.hjug.git.ChangePronenessRanker;
import org.hjug.git.GitLogReader;
Expand All @@ -33,14 +32,14 @@
import org.jgrapht.graph.DefaultWeightedEdge;

@Slf4j
public class CostBenefitCalculator {
public class CostBenefitCalculator implements AutoCloseable {

private final Map<String, AsSubgraph<String, DefaultWeightedEdge>> renderedSubGraphs = new HashMap<>();

private Report report;
private String repositoryPath;
private final GitLogReader gitLogReader = new GitLogReader();
private Repository repository = null;
private GitLogReader gitLogReader;

private final ChangePronenessRanker changePronenessRanker;
private final JavaProjectParser javaProjectParser = new JavaProjectParser();

Expand All @@ -52,16 +51,22 @@ public CostBenefitCalculator(String repositoryPath) {

log.info("Initiating Cost Benefit calculation");
try {
repository = gitLogReader.gitRepository(new File(repositoryPath));
for (String file :
gitLogReader.listRepositoryContentsAtHEAD(repository).keySet()) {
log.info("Files at HEAD: {}", file);
}
gitLogReader = new GitLogReader(new File(repositoryPath));
// repository = gitLogReader.gitRepository(new File(repositoryPath));
// for (String file :
// gitLogReader.listRepositoryContentsAtHEAD(repository).keySet()) {
// log.info("Files at HEAD: {}", file);
// }
} catch (IOException e) {
log.error("Failure to access Git repository", e);
}

changePronenessRanker = new ChangePronenessRanker(repository, gitLogReader);
changePronenessRanker = new ChangePronenessRanker(gitLogReader);
}

@Override
public void close() throws Exception {
gitLogReader.close();
}

public List<RankedCycle> runCycleAnalysis(String outputDirectoryPath, boolean renderImages) {
Expand Down Expand Up @@ -148,7 +153,7 @@ public List<RankedCycle> runCycleAnalysis(String outputDirectoryPath, boolean re

private boolean isDuplicateSubGraph(AsSubgraph<String, DefaultWeightedEdge> subGraph, String vertex) {
if (!renderedSubGraphs.isEmpty()) {
for (AsSubgraph<String, DefaultWeightedEdge> renderedSubGraph : renderedSubGraphs.values()) {
for (AsSubgraph<String, DefaultWeightedEdge> renderedSubGraph : renderedSubGraphs.values()) {
if (renderedSubGraph.vertexSet().size() == subGraph.vertexSet().size()
&& renderedSubGraph.edgeSet().size()
== subGraph.edgeSet().size()
Expand Down Expand Up @@ -176,7 +181,7 @@ public void runPmdAnalysis() throws IOException {
log.info("files to be scanned: " + Paths.get(repositoryPath));

try (Stream<Path> files = Files.walk(Paths.get(repositoryPath))) {
files.forEach(file -> pmd.files().addFile(file));
files.filter(Files::isRegularFile).forEach(file -> pmd.files().addFile(file));
}

report = pmd.performAnalysisAndCollectReport();
Expand Down Expand Up @@ -231,25 +236,23 @@ private List<GodClass> getGodClasses() {
}

<T extends Disharmony> List<ScmLogInfo> getRankedChangeProneness(List<T> disharmonies) {
List<ScmLogInfo> scmLogInfos = new ArrayList<>();
log.info("Calculating Change Proneness");
for (Disharmony disharmony : disharmonies) {
String path = disharmony.getFileName();
ScmLogInfo scmLogInfo = null;
try {
scmLogInfo = gitLogReader.fileLog(repository, path);
log.info("Successfully fetched scmLogInfo for {}", scmLogInfo.getPath());
} catch (GitAPIException | IOException e) {
log.error("Error reading Git repository contents.", e);
} catch (NullPointerException e) {
log.error("Encountered nested class in a class containing a violation. Class: {}", path);
}

if (null != scmLogInfo) {
log.info("adding {}", scmLogInfo.getPath());
scmLogInfos.add(scmLogInfo);
}
}
List<ScmLogInfo> scmLogInfos = disharmonies.parallelStream()
.map(disharmony -> {
String path = disharmony.getFileName();
ScmLogInfo scmLogInfo = null;
try {
scmLogInfo = gitLogReader.fileLog(path);
log.info("Successfully fetched scmLogInfo for {}", scmLogInfo.getPath());
} catch (GitAPIException | IOException e) {
log.error("Error reading Git repository contents.", e);
} catch (NullPointerException e) {
log.error("Encountered nested class in a class containing a violation. Class: {}", path);
}
return scmLogInfo;
})
.collect(Collectors.toList());

changePronenessRanker.rankChangeProneness(scmLogInfos);
return scmLogInfos;
Expand Down
Loading
Loading