when i’m trying to get the commits diff, i’m getting a NullPointerException.
The strange thing i’m stuggling since 4 days now it’s when i’m in quarkus dev mode (when i execute my app from intellij) i have no problems, But in prod mode there is problems (btw I’m cloning 5 projects, and it’s always the same 2 of them which are throwing the errors)
The method :
private void processDifferences(Repository repo, RevCommit revCommit, Git git, String gitBranch) {
var stats = CommitEntity.find("commitSHA", revCommit.getId().getName()).firstResultOptional();
if (stats.isPresent()) {
return;
}
try {
Log.info("Collect stats for commit :" + revCommit.getId().getName());
// To collect statistics from a commit
DiffFormatter diffFormatter = new DiffFormatter(DisabledOutputStream.INSTANCE);
RevCommit parentCommit = revCommit.getParentCount() > 0 ? revCommit.getParent(0) : null;
diffFormatter.setRepository(repo);
diffFormatter.setDetectRenames(true);
List<DiffEntry> diffStats = diffFormatter.scan(parentCommit, revCommit);
var insertions = 0;
var deletions = 0;
for (DiffEntry diffStat : diffStats) {
DiffFormatter diffFormatterForDiffStat = new DiffFormatter(DisabledOutputStream.INSTANCE);
diffFormatterForDiffStat.setRepository(repo);
diffFormatterForDiffStat.setDetectRenames(true);
var edits = diffFormatterForDiffStat.toFileHeader(diffStat).toEditList();
for (var edit : edits) {
insertions += edit.getLengthB();
deletions += edit.getLengthA();
}
}
saveStatistics(git, revCommit, gitBranch, revCommit.getId(), insertions, deletions);
} catch (Exception e) {
Log.error("Error while processing commit", e);
}
}
The stacktrace :
2024-05-24 14:58:14,435 INFO [com.elo.pro.ser.GitService] (executor-thread-1) Collect stats for commit :1fc08938cb3df41e41f9f6b24b060bbbabe6a252
2024-05-24 14:58:14,447 ERROR [com.elo.pro.ser.GitService] (executor-thread-1) Error while processing commit: java.lang.NullPointerException
at org.eclipse.jgit.internal.storage.file.WindowCache.evict(WindowCache.java:663)
at org.eclipse.jgit.internal.storage.file.WindowCache.getOrLoad(WindowCache.java:623)
at org.eclipse.jgit.internal.storage.file.WindowCache.get(WindowCache.java:387)
at org.eclipse.jgit.internal.storage.file.WindowCursor.pin(WindowCursor.java:321)
at org.eclipse.jgit.internal.storage.file.WindowCursor.inflate(WindowCursor.java:283)
at org.eclipse.jgit.internal.storage.file.Pack.decompress(Pack.java:387)
at org.eclipse.jgit.internal.storage.file.Pack.load(Pack.java:907)
at org.eclipse.jgit.internal.storage.file.Pack.get(Pack.java:290)
at org.eclipse.jgit.internal.storage.file.PackDirectory.open(PackDirectory.java:228)
at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openPackedObject(ObjectDirectory.java:412)
at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openPackedFromSelfOrAlternate(ObjectDirectory.java:375)
at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openObjectWithoutRestoring(ObjectDirectory.java:365)
at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openObject(ObjectDirectory.java:350)
at org.eclipse.jgit.internal.storage.file.WindowCursor.open(WindowCursor.java:133)
at org.eclipse.jgit.diff.ContentSource$ObjectReaderSource.open(ContentSource.java:135)
at org.eclipse.jgit.diff.ContentSource$Pair.open(ContentSource.java:300)
at org.eclipse.jgit.diff.DiffFormatter.open(DiffFormatter.java:1080)
at org.eclipse.jgit.diff.DiffFormatter.createFormatResult(DiffFormatter.java:1007)
at org.eclipse.jgit.diff.DiffFormatter.toFileHeader(DiffFormatter.java:970)
at com.elosi.projectsboard.service.GitService.processDifferences(GitService.java:240)
at com.elosi.projectsboard.service.GitService.processBranch(GitService.java:199)
at com.elosi.projectsboard.service.GitService.processProject(GitService.java:174)
at com.elosi.projectsboard.service.GitService_Subclass.processProject$$superforward(Unknown Source)
at com.elosi.projectsboard.service.GitService_Subclass$$function$$2.apply(Unknown Source)
at io.quarkus.arc.impl.AroundInvokeInvocationContext.proceed(AroundInvokeInvocationContext.java:73)
at io.quarkus.arc.impl.AroundInvokeInvocationContext.proceed(AroundInvokeInvocationContext.java:62)
at io.quarkus.narayana.jta.runtime.interceptor.TransactionalInterceptorBase.invokeInOurTx(TransactionalInterceptorBase.java:136)
at io.quarkus.narayana.jta.runtime.interceptor.TransactionalInterceptorBase.invokeInOurTx(TransactionalInterceptorBase.java:107)
at io.quarkus.narayana.jta.runtime.interceptor.TransactionalInterceptorRequired.doIntercept(TransactionalInterceptorRequired.java:38)
at io.quarkus.narayana.jta.runtime.interceptor.TransactionalInterceptorBase.intercept(TransactionalInterceptorBase.java:61)
at io.quarkus.narayana.jta.runtime.interceptor.TransactionalInterceptorRequired.intercept(TransactionalInterceptorRequired.java:32)
at io.quarkus.narayana.jta.runtime.interceptor.TransactionalInterceptorRequired_Bean.intercept(Unknown Source)
at io.quarkus.arc.impl.InterceptorInvocation.invoke(InterceptorInvocation.java:42)
at io.quarkus.arc.impl.AroundInvokeInvocationContext.perform(AroundInvokeInvocationContext.java:30)
at io.quarkus.arc.impl.InvocationContexts.performAroundInvoke(InvocationContexts.java:27)
at com.elosi.projectsboard.service.GitService_Subclass.processProject(Unknown Source)
at com.elosi.projectsboard.service.GitService_ClientProxy.processProject(Unknown Source)
at com.elosi.projectsboard.schedule.Scheduler.handleUpdateStatistics(Scheduler.java:43)
at com.elosi.projectsboard.schedule.Scheduler_ClientProxy.handleUpdateStatistics(Unknown Source)
at com.elosi.projectsboard.schedule.Scheduler_ScheduledInvoker_handleUpdateStatistics_c42f06a4b7ff3946bd41561d9caa5219ebe82990.invokeBean(Unknown Source)
at io.quarkus.scheduler.common.runtime.DefaultInvoker.invoke(DefaultInvoker.java:24)
at io.quarkus.scheduler.common.runtime.StatusEmitterInvoker.invoke(StatusEmitterInvoker.java:35)
at io.quarkus.scheduler.runtime.SimpleScheduler$ScheduledTask.doInvoke(SimpleScheduler.java:443)
at io.quarkus.scheduler.runtime.SimpleScheduler$ScheduledTask$2.call(SimpleScheduler.java:425)
at io.quarkus.scheduler.runtime.SimpleScheduler$ScheduledTask$2.call(SimpleScheduler.java:422)
at io.vertx.core.impl.ContextImpl.lambda$executeBlocking$0(ContextImpl.java:178)
at io.vertx.core.impl.ContextInternal.dispatch(ContextInternal.java:279)
at io.vertx.core.impl.ContextImpl.lambda$internalExecuteBlocking$2(ContextImpl.java:210)
at io.quarkus.vertx.core.runtime.VertxCoreRecorder$14.runWith(VertxCoreRecorder.java:604)
at org.jboss.threads.EnhancedQueueExecutor$Task.doRunWith(EnhancedQueueExecutor.java:2516)
at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2495)
at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1521)
at org.jboss.threads.DelegatingRunnable.run(DelegatingRunnable.java:11)
at org.jboss.threads.ThreadLocalResettingRunnable.run(ThreadLocalResettingRunnable.java:11)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at [email protected]/java.lang.Thread.runWith(Thread.java:1596)
at [email protected]/java.lang.Thread.run(Thread.java:1583)
at org.graalvm.nativeimage.builder/com.oracle.svm.core.thread.PlatformThreads.threadStartRoutine(PlatformThreads.java:896)
at org.graalvm.nativeimage.builder/com.oracle.svm.core.thread.PlatformThreads.threadStartRoutine(PlatformThreads.java:872)
So i’m working on a Quarkus Java app using JGit to insert into a postgresql database data like users, commits, branches, projects via a scheduled method.
Plugged to the postgres i got also a grafana.
My scheduled method is reading a file “paths” to get projects to clone
What i expected is cloning every projects and insert in database the data from the .git folder (commits and their diff, users from the commit, project and branches, …).
In local it works good but not when i’m building it in prod with quarkus.
I’ll also provide my Docker-Compose and a DockerFile.
The scheduled method :
@Scheduled(cron = "0 0/2 * * * ?")
public void handleUpdateStatistics() {
gitConfig.init();
Log.info("Updating statistics");
try (BufferedReader reader = new BufferedReader(new FileReader("paths"))) {
String line = reader.readLine();
while (line != null) {
try {
Git git = gitService.cloneOrPullRepository(line, storagePath, gitConfig);
gitService.processProject(git, storagePath);
git.close();
} catch (IOException | GitAPIException | NullPointerException e) {
Log.error("Error while processing project: " + line + " | cause : " + e.getMessage());
}
line = reader.readLine();
}
} catch (IOException e) {
Log.error("Error while reading file 'paths': " + e.getMessage());
}
Log.info("Statistics updated");
}
Here practically all my GitService :
/**
* This method clone or pull the repositories from Git.
* It also save projects name and group into the database.
*
* @param url the url of the repository
* @param storagePath where the repository is physically saved
* @param config the git configuration
* @return a repository as {@link Git}
* @throws IOException exception thrown when there is a problem to read or save files.
* @throws GitAPIException exception thrown when there is an issue to connect or use Git.
*/
@Transactional
public Git cloneOrPullRepository(String url, String storagePath, GitConfig config) throws IOException, GitAPIException {
String repoPath = String.format("%s/%s", storagePath, getRepositoryName(url));
File repoDir = new File(repoPath);
try {
if (!repoDir.exists()) {
createDirectory(repoDir);
return cloneRepository(url, repoDir, config);
} else {
return pullRepository(repoDir, config);
}
} catch (GitAPIException | IOException e) {
handleCloneOrPullException(repoDir, e);
return null;
}
}
/**
* This method clone a git repository into a folder (see 'STORAGE_PATH' in your .env file)
* @param url the git url of the repository
* @param repoDir the directory path
* @param config the Git configuration
* @return a git repository as {@link Git}
* @throws IOException if there is a problem with filesystem to create the directory
* @throws GitAPIException if there is a problem with git directly
*/
private Git cloneRepository(String url, File repoDir, GitConfig config) throws GitAPIException, IOException {
Log.info("n");
Log.info("Cloning new repo: " + repoDir.getAbsolutePath());
Git repository;
try {
Log.info("Url : " + String.format("%s/%s", config.getBaseurl(), url));
Log.info("Directory : " + repoDir);
Log.info("Git Credentials : " + config);
repository = Git.cloneRepository()
.setCredentialsProvider(config.getCredentialsProvider())
.setDirectory(repoDir)
.setNoCheckout(true)
.setURI(String.format("%s/%s", config.getBaseurl(), url))
.call();
Log.info("Cloned new repo: " + repoDir.getAbsolutePath());
repository.fetch().setCredentialsProvider(config.getCredentialsProvider()).call();
saveProjectDatabase(url);
return repository;
} catch (GitAPIException | NullPointerException e) {
handleCloneOrPullException(repoDir, e);
return null;
}
}
/**
* This method pull a repository already cloned by this app.
* @param repoDir the directory path
* @param config the Git configuration
* @return a git repository as {@link Git}
* @throws IOException if there is a problem with filesystem to create the directory
* @throws GitAPIException if there is a problem with git directly
*/
private Git pullRepository(File repoDir, GitConfig config) throws IOException, GitAPIException {
Log.info("n");
Log.info("Pulling repo: " + repoDir.getAbsolutePath());
Git repository = Git.open(repoDir);
repository.pull().setCredentialsProvider(config.getCredentialsProvider()).call();
Log.info("Pulled repo: " + repoDir.getAbsolutePath());
return repository;
}
/**
* This method handle errors when pulling or cloning a git repo thanks to JGit fails.
* After the exception is logged, this method delete the folder which has failed if exists.
* @param repoDir the directory path
* @param e the exception handled
* @throws IOException if there is a problem with filesystem to create the directory
*/
private void handleCloneOrPullException(File repoDir, Exception e) throws IOException {
Log.error("Error during clone or pull repo : " + repoDir.getAbsolutePath() + " | cause : " + e.getMessage(), e);
if (repoDir.exists()) {
Files.walkFileTree(repoDir.toPath(), new DeleteFileVisitor());
}
}
/**
* This method create directory from a File
* @param dir the directory path
* @throws IOException if there is a problem with filesystem to create the directory
*/
private void createDirectory(File dir) throws IOException {
if (!dir.exists()) {
boolean isCreated = dir.mkdirs();
if (!isCreated) {
throw new IOException("Failed to create directory: " + dir.getAbsolutePath());
}
}
}
/**
* This method check if there is differences from the origin to the local for a project.
*
* @param git the {@link Git} configuration
* @param storagePath the physical storage of repositories
* @throws IOException exception thrown when there is a problem to read or save files.
* @throws GitAPIException exception thrown when there is an issue to connect or use Git.
*/
@Transactional
public void processProject(Git git, String storagePath) throws GitAPIException, IOException {
Repository repo = git.getRepository();
// Process the main branch
processBranch(repo, git, repo.getBranch(), storagePath, null);
// Process other branches
for (Ref ref : git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE).call()) {
if (ref.getName().contains(repo.getBranch())) {
Log.info("Skipping branch " + ref.getName());
continue;
}
Log.info("Processing branch " + ref.getName());
processBranch(repo, git, ref.getName(), storagePath, ref);
}
// TODO maybe add a method deleting branches in databases which are deleted on origin | OR |
// Update a column 'status' in branches table to make it 'deleted / merged / ...'
}
/**
* This method work with {@link #processProject(Git, String)}, for a project it calculates differences for a branch
* It also save the branches into the database.
*
* @param repo the git repository
* @param git the git configuration
* @param branchName the branch name
* @param storagePath the physical storage of repositories
* @param ref the ref of the branch in the distant git server
*/
private void processBranch(Repository repo, Git git, String branchName, String storagePath, Ref ref) {
saveBranchDatabase(git, branchName, storagePath);
// Process commits for the branch
try {
final var logs = ref != null ? git.log().add(ref.getObjectId()).call() : git.log().call();
for (RevCommit revCommit : logs) {
processDifferences(repo, revCommit, git, branchName);
}
} catch (Exception e) {
Log.error("Error processing branch " + branchName + ": " + e.getMessage());
}
}
/**
* This method is calculating the differences from a commit to his parent
*
* @param repo the git {@link Repository}
* @param revCommit the commit as {@link RevCommit}
* @param git the git configuration
* @param gitBranch the git branch
*/
private void processDifferences(Repository repo, RevCommit revCommit, Git git, String gitBranch) {
var stats = CommitEntity.find("commitSHA", revCommit.getId().getName()).firstResultOptional();
if (stats.isPresent()) {
return;
}
try {
Log.info("Collect stats for commit :" + revCommit.getId().getName());
// To collect statistics from a commit
DiffFormatter diffFormatter = new DiffFormatter(DisabledOutputStream.INSTANCE);
RevCommit parentCommit = revCommit.getParentCount() > 0 ? revCommit.getParent(0) : null;
diffFormatter.setRepository(repo);
diffFormatter.setDetectRenames(true);
List<DiffEntry> diffStats = diffFormatter.scan(parentCommit, revCommit);
var insertions = 0;
var deletions = 0;
for (DiffEntry diffStat : diffStats) {
DiffFormatter diffFormatterForDiffStat = new DiffFormatter(DisabledOutputStream.INSTANCE);
diffFormatterForDiffStat.setRepository(repo);
diffFormatterForDiffStat.setDetectRenames(true);
var edits = diffFormatterForDiffStat.toFileHeader(diffStat).toEditList();
for (var edit : edits) {
insertions += edit.getLengthB();
deletions += edit.getLengthA();
}
}
saveStatistics(git, revCommit, gitBranch, revCommit.getId(), insertions, deletions);
} catch (Exception e) {
Log.error("Error while processing commit", e);
}
}
Docker-Compose File :
services:
postgres:
image: postgres:13
container_name: postgres
env_file: .env.prod
environment:
POSTGRES_USER: ${PSQL_USERNAME}
POSTGRES_PASSWORD: ${PSQL_PASSWD}
POSTGRES_DB: projectsboard
volumes:
- postgres_data:/var/lib/postgresql/data
ports:
- "5432:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${PSQL_USERNAME}"]
interval: 5s
timeout: 10s
retries: 5
grafana:
image: grafana/grafana:latest
container_name: grafana
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin
ports:
- "3000:3000"
depends_on:
postgres:
condition: service_healthy
quarkus:
build:
context: .
dockerfile: src/main/docker/Dockerfile.native
container_name: quarkus
env_file: .env.prod
environment:
- GIT_USERNAME=${GIT_USERNAME}
- GIT_PASSWORD=${GIT_PASSWORD}
- GIT_BASEURL=${GIT_BASEURL}
- STORAGE_PATH=${STORAGE_PATH}
- CONFIG_PATHSFILE=${CONFIG_PATHSFILE}
- DEFAULT_USER_MAIL=${DEFAULT_USER_MAIL}
- PSQL_USERNAME=${PSQL_USERNAME}
- PSQL_PASSWD=${PSQL_PASSWD}
- PSQL_HOST=postgres
- PSQL_PORT=5432
- PSQL_DB=projectsboard
ports:
- "8080:8080"
depends_on:
postgres:
condition: service_healthy
volumes:
- storage_data:/work/storage
deploy:
resources:
limits:
memory: 2g
reservations:
memory: 1g
volumes:
postgres_data:
storage_data:
Dockerfile.native :
####
# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode.
#
# Before building the container image run:
#
# ./mvnw package -Dnative
#
# Then, build the image with:
#
# docker build -f src/main/docker/Dockerfile.native -t quarkus/projectboard .
#
# Then run the container using:
#
# docker run -i --rm -p 8080:8080 quarkus/projectboard
#
###
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9
WORKDIR /work/
RUN chown root /work
&& chmod "g+rwX" /work
&& chown root:root /work
COPY --chown=root:root target/*-runner /work/application
COPY --chown=root:root paths /work/paths
EXPOSE 8080
USER root
HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 CMD curl -f http://localhost:8080/health || exit 1
ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"]