Skip to content

qprofiler1 / qvisualise - handle new fastq header #336

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 17, 2023
Merged

qprofiler1 / qvisualise - handle new fastq header #336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8b22945
fix(qvisualise/qprofiler): deal with new fastq header and make qvis m…
Jul 14, 2023
3353fd1
refactor(qprofiler qvisualise): adding comments and tweaking logging …
Jul 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 22 additions & 22 deletions qprofiler/src/org/qcmg/qprofiler/fastq/FastqSummaryReport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,12 @@ public class FastqSummaryReport extends SummaryReport {
private static final Integer i = Integer.MAX_VALUE;

//SEQ
private final SummaryByCycleNew2<Character> seqByCycle = new SummaryByCycleNew2<Character>(c, 512);
private Map<Integer, AtomicLong> seqLineLengths = null;
private final SummaryByCycleNew2<Character> seqByCycle = new SummaryByCycleNew2<>(c, 512);
private final QCMGAtomicLongArray seqBadReadLineLengths = new QCMGAtomicLongArray(128);
private final KmersSummary kmersSummary = new KmersSummary( KmersSummary.MAX_KMERS ); //default use biggest mers length

//QUAL
private final SummaryByCycleNew2<Integer> qualByCycleInteger = new SummaryByCycleNew2<Integer>(i, 512);
private Map<Integer, AtomicLong> qualLineLengths = null;
private final SummaryByCycleNew2<Integer> qualByCycleInteger = new SummaryByCycleNew2<>(i, 512);
private final QCMGAtomicLongArray qualBadReadLineLengths = new QCMGAtomicLongArray(128);

// Header info
Expand Down Expand Up @@ -114,16 +112,16 @@ public void toXml(Element parent) {
SummaryReportUtils.lengthMapToXml(readNameElement, "QUAL_HEADERS", qualHeaders);

// create the length maps here from the cycles objects
seqLineLengths = SummaryByCycleUtils.getLengthsFromSummaryByCycle(seqByCycle, getRecordsParsed());
qualLineLengths = SummaryByCycleUtils.getLengthsFromSummaryByCycle(qualByCycleInteger, getRecordsParsed());
Map<Integer, AtomicLong> seqLineLengths = SummaryByCycleUtils.getLengthsFromSummaryByCycle(seqByCycle, getRecordsParsed());
Map<Integer, AtomicLong> qualLineLengths = SummaryByCycleUtils.getLengthsFromSummaryByCycle(qualByCycleInteger, getRecordsParsed());

// SEQ
Element seqElement = createSubElement(element, "SEQ");
seqByCycle.toXml(seqElement, "BaseByCycle");
SummaryReportUtils.lengthMapToXmlTallyItem(seqElement, "LengthTally", seqLineLengths);
SummaryReportUtils.lengthMapToXml(seqElement, "BadBasesInReads", seqBadReadLineLengths);

kmersSummary.toXml(seqElement,kmersSummary.MAX_KMERS);
kmersSummary.toXml(seqElement, KmersSummary.MAX_KMERS);
kmersSummary.toXml(seqElement,1); //add 1-mers
kmersSummary.toXml(seqElement,2); //add 2-mers
kmersSummary.toXml(seqElement,3); //add 3-mers
Expand All @@ -137,11 +135,6 @@ public void toXml(Element parent) {
}
}

/**
* Reads a row from the text file and returns it as a string
*
* @return next row in file
*/
public void parseRecord(FastqRecord record) {
if (null != record) {

Expand All @@ -158,12 +151,19 @@ public void parseRecord(FastqRecord record) {
byte[] readBases = record.getReadString().getBytes();
SummaryByCycleUtils.parseCharacterSummary(seqByCycle, readBases, reverseStrand);
SummaryReportUtils.tallyBadReadsAsString(readBases, seqBadReadLineLengths);
kmersSummary.parseKmers( readBases, false ); //fastq base are all orignal forward
kmersSummary.parseKmers( readBases, false ); //fastq base are all original forward

// header stuff
if (record.getReadName().contains(":")) {
String [] headerDetails = TabTokenizer.tokenize(record.getReadName(), ':');
if (null != headerDetails && headerDetails.length > 0) {

String headerToUse = record.getReadName();
int spaceCount = StringUtils.getCount(headerToUse, ' ');
if (spaceCount == 2) {
headerToUse = TabTokenizer.tokenize(headerToUse, ' ')[1];
}

if (headerToUse.contains(":")) {
String [] headerDetails = TabTokenizer.tokenize(headerToUse, ':');
if (headerDetails.length > 0) {

//if length is equal to 10, we have the classic Casava 1.8 format

Expand All @@ -180,7 +180,7 @@ public void parseRecord(FastqRecord record) {
// 13051 - x
// 2071 - y
// 2 - 2nd in pair
if (record.getReadName().contains(" ")) {
if (headerToUse.contains(" ")) {
parseFiveElementHeaderWithSpaces(headerDetails);
} else {
parseFiveElementHeaderNoSpaces(headerDetails);
Expand Down Expand Up @@ -231,13 +231,13 @@ public void parseRecord(FastqRecord record) {
filteredN.incrementAndGet();
}

// skip control bit for now
// skip the control bit for now

// indexes
if (headerLength > 9) {
key = headerDetails[9];
updateMap(indexes, key);
} // thats it!!
} // that's it!!
}
}
}
Expand Down Expand Up @@ -292,18 +292,18 @@ void parseFiveElementHeaderWithSpaces(String [] params) {
// split by space
String [] firstElementParams = params[0].split(" ");
if (firstElementParams.length != 2) {
throw new UnsupportedOperationException("Incorrect header format encountered in parseFiveElementHeader. Expected '@ERR091788.3104 HSQ955_155:2:1101:13051:2071/2' but recieved: " + Arrays.deepToString(params));
throw new IllegalArgumentException("Incorrect header format encountered in parseFiveElementHeaderWithSpaces. Expected a space (e.g. @ERR091788.3104 HSQ955_155) in the first element in the array, but received: " + Arrays.deepToString(params));
}
String [] machineAndReadPosition = firstElementParams[0].split("\\.");
if (machineAndReadPosition.length != 2) {
throw new UnsupportedOperationException("Incorrect header format encountered in parseFiveElementHeader. Expected '@ERR091788.3104 HSQ955_155:2:1101:13051:2071/2' but recieved: " + Arrays.deepToString(params));
throw new IllegalArgumentException("Incorrect header format encountered in parseFiveElementHeaderWithSpaces. Expected a single dot (e.g. @ERR091788.3104 HSQ955_155) in the first part of the first element in the array, but received: " + Arrays.deepToString(params));
}

updateMap(instruments, machineAndReadPosition[0]);

String [] flowCellAndRunId = firstElementParams[1].split("_");
if (flowCellAndRunId.length != 2) {
throw new UnsupportedOperationException("Incorrect header format encountered in parseFiveElementHeader. Expected '@ERR091788.3104 HSQ955_155:2:1101:13051:2071/2' but recieved: " + Arrays.deepToString(params));
throw new IllegalArgumentException("Incorrect header format encountered in parseFiveElementHeaderWithSpaces. Expected a single underscore (e.g. @ERR091788.3104 HSQ955_155) in the second part of the first element in the array but received: " + Arrays.deepToString(params));
}

updateMap(flowCellIds, flowCellAndRunId[0]);
Expand Down
124 changes: 107 additions & 17 deletions qprofiler/test/org/qcmg/qprofiler/fastq/FastqSummaryReportTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,51 +1,141 @@
package org.qcmg.qprofiler.fastq;

import static org.junit.Assert.assertEquals;
import htsjdk.samtools.fastq.FastqRecord;

import org.junit.Ignore;
import org.junit.Test;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

public class FastqSummaryReportTest {

@Test
public void parseRecordHeader() {
FastqRecord rec = new FastqRecord("@ERR091788.1 HSQ955_155:2:1101:1473:2037/1",
FastqRecord rec = new FastqRecord("ERR091788.1 HSQ955_155:2:1101:1473:2037/1",
"GGGCANCCAGCAGCCCTCGGGGCTTCTCTGTTTATGGAGTAGCCATTCTCGTATCCTTCTACTTTCTTAAACTTTCTTTCACTTACAAAAAAATAGTGGA",
"+",
"",
"<@@DD#2AFFHHH<FHFF@@FEG@DF?BF4?FFGDIBC?B?=FHIEFHGGG@CGHIIHDHFHFECDEEEECCCCCCAC@CCC>CCCCCCBBBBAC>:@<C");

FastqSummaryReport report = new FastqSummaryReport();
report.parseRecord(rec);
assertEquals(1, report.getRecordsParsed());
assertEquals(1, report.instruments.get("@ERR091788").intValue());
assertEquals(1, report.instruments.get("ERR091788").intValue());
assertEquals(1, report.flowCellIds.get("HSQ955").intValue());
assertEquals(1, report.flowCellLanes.get("2").intValue());
assertEquals(1, report.tileNumbers.get(1101).intValue());
assertEquals(1, report.firstInPair.intValue());
assertEquals(0, report.secondInPair.intValue());

}

@Ignore // may need to cater for this in the future...
public void parseAnotherRecordHeader() {
FastqRecord rec = new FastqRecord("@SRR001666.1 071112_SLXA-EAS1_s_7:5:1:817:345 length=36",
@Test // may need to cater for this in the future...
public void parseHeaderDoubleSpace2() {
FastqRecord rec = new FastqRecord("SRR001666.1 071112_SLXA-EAS1_s_7:5:1:817:345 length=36",
"GGGCANCCAGCAGCCCTCGGGGCTTCTCTGTTTATGGAGTAGCCATTCTCGTATCCTTCTACTTTCTTAAACTTTCTTTCACTTACAAAAAAATAGTGGA",
"+",
"",
"<@@DD#2AFFHHH<FHFF@@FEG@DF?BF4?FFGDIBC?B?=FHIEFHGGG@CGHIIHDHFHFECDEEEECCCCCCAC@CCC>CCCCCCBBBBAC>:@<C");

FastqSummaryReport report = new FastqSummaryReport();
report.parseRecord(rec);
assertEquals(1, report.getRecordsParsed());
assertEquals(1, report.instruments.get("@ERR091788").intValue());
assertEquals(1, report.flowCellIds.get("HSQ955").intValue());
assertEquals(1, report.flowCellLanes.get("2").intValue());
assertEquals(1, report.instruments.get("071112").intValue());
assertEquals(1, report.runIds.get("SLXA-EAS1_s_7").intValue());
assertEquals(1, report.flowCellLanes.get("5").intValue());
assertEquals(1, report.tileNumbers.get(1).intValue());
assertEquals(0, report.firstInPair.intValue());
assertEquals(0, report.secondInPair.intValue());
}
@Test // may need to cater for this in the future...
public void parseHeaderDoubleSpace() {

/*
@SRR14585604.19092 A00805:41:HMJJWDRXX:1:1101:15329:20181 length=101
TGCATTGTGTCAAAAGAAATTTCCTTATTTTCTACTGCCATTCCCATAAAAGTAAGTAGTCTCATTTTTGACATATTCTGTTCATGTAACAGGCCAAGTTA
+SRR14585604.19092 A00805:41:HMJJWDRXX:1:1101:15329:20181 length=101
:::FF:F,:F,FF:F:FFF:FFF,FFFFF,FF:FFF,F:F:,F,:FFF:FF:FF:F,F,::F::FF,FF,,:F,F,FFF,FFF:,,FFFFF,F:FF,FF,F
*/
FastqRecord rec = new FastqRecord("SRR14585604.19092 A00805:41:HMJJWDRXX:1:1101:15329:20181 length=101",
"TGCATTGTGTCAAAAGAAATTTCCTTATTTTCTACTGCCATTCCCATAAAAGTAAGTAGTCTCATTTTTGACATATTCTGTTCATGTAACAGGCCAAGTTA",
"SRR14585604.19092 A00805:41:HMJJWDRXX:1:1101:15329:20181 length=101",
":::FF:F,:F,FF:F:FFF:FFF,FFFFF,FF:FFF,F:F:,F,:FFF:FF:FF:F,F,::F::FF,FF,,:F,F,FFF,FFF:,,FFFFF,F:FF,FF,F");

FastqSummaryReport report = new FastqSummaryReport();
report.parseRecord(rec);
assertEquals(1, report.getRecordsParsed());
assertEquals(1, report.instruments.get("A00805").intValue());
assertEquals(1, report.flowCellIds.get("HMJJWDRXX").intValue());
assertEquals(1, report.flowCellLanes.get("1").intValue());
assertEquals(1, report.tileNumbers.get(1101).intValue());
assertEquals(0, report.firstInPair.intValue());
assertEquals(0, report.secondInPair.intValue());
}

@Test
public void parseHeaderSingleSpace() {
/*
@VH01336:23:AAC37HWHV:1:1101:18459:1000 1:N:0:GGGGGGGG+AGATCTCG
GTCCAGTTGCATTTTAGTAAGCTCTTTTTGATTCTCAAATCCGGCGTCAACCATACCAGCAGAGGAAGCATCAGCACCAGCACGCTCCCAAGCATTAAGCT
+
CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC;CCCCCCCCC;;CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC-CCCCCCCC
*/
FastqRecord rec = new FastqRecord("VH01336:23:AAC37HWHV:1:1101:18459:1000 1:N:0:GGGGGGGG+AGATCTCG",
"GTCCAGTTGCATTTTAGTAAGCTCTTTTTGATTCTCAAATCCGGCGTCAACCATACCAGCAGAGGAAGCATCAGCACCAGCACGCTCCCAAGCATTAAGCT",
"",
"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC;CCCCCCCCC;;CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC-CCCCCCCC");

FastqSummaryReport report = new FastqSummaryReport();
report.parseRecord(rec);
assertEquals(1, report.getRecordsParsed());
assertEquals(1, report.instruments.get("VH01336").intValue());
assertEquals(1, report.flowCellIds.get("AAC37HWHV").intValue());
assertEquals(1, report.flowCellLanes.get("1").intValue());
assertEquals(1, report.tileNumbers.get(1101).intValue());
assertEquals(1, report.firstInPair.intValue());
assertEquals(0, report.secondInPair.intValue());

}



@Test
public void parseHeaderNoSpace() {
/*
@HWI-ST590:2:1201:12570:134058#0
AATAGTCCTAACGTTCTACATAACTTCAAGTAGTAAAATTCACCATCCTCT
+
:BC8?ABCEBEB9CEBFB@BC;>BFD=DE?B;@DBDED?DCD?BDDDBBBB
*/
FastqRecord rec = new FastqRecord("HWI-ST590:2:1201:12570:134058#0",
"AATAGTCCTAACGTTCTACATAACTTCAAGTAGTAAAATTCACCATCCTCT",
"",
":BC8?ABCEBEB9CEBFB@BC;>BFD=DE?B;@DBDED?DCD?BDDDBBBB");

FastqSummaryReport report = new FastqSummaryReport();
report.parseRecord(rec);
assertEquals(1, report.getRecordsParsed());
assertEquals(1, report.instruments.get("HWI-ST590").intValue());
assertTrue(report.flowCellIds.isEmpty());
assertEquals(1, report.flowCellLanes.get("2").intValue());
assertEquals(1, report.tileNumbers.get(1201).intValue());
assertEquals(0, report.firstInPair.intValue());
assertEquals(0, report.secondInPair.intValue());
}
@Test
public void parseHeaderNoSpace2() {
/*
@V350046278L1C001R00100004433/2
CGCTGAAAATTGAAAGCCCGCTTGGGATAAGTGACATTAAGAACTGGCACCGACTGCAGAACCGCAATTTCCAGTTGACGCTAAGTGGGGGCTTATTTAGCACCCAGCTCTGTTTGCCAACACCCCCTGGGCATGAGAGCTCCCCAAGGG
+
HGGCGEE<DDDH<EAHBFGGGDBHHBB;C:HCGBEEBA@8HEDGAFGECFGG,1BH?@G)-C@EB?D/C6>GDHBBF(EH:7>>G@GH?G?F@?6<CB?B=DEC:>>CBE?G???BBG<.F:E?CFD?@?A:#5E>5BE/>BFFD+$E,>
*/
FastqRecord rec = new FastqRecord("V350046278L1C001R00100004433/2",
"CGCTGAAAATTGAAAGCCCGCTTGGGATAAGTGACATTAAGAACTGGCACCGACTGCAGAACCGCAATTTCCAGTTGACGCTAAGTGGGGGCTTATTTAGCACCCAGCTCTGTTTGCCAACACCCCCTGGGCATGAGAGCTCCCCAAGGG",
"",
"HGGCGEE<DDDH<EAHBFGGGDBHHBB;C:HCGBEEBA@8HEDGAFGECFGG,1BH?@G)-C@EB?D/C6>GDHBBF(EH:7>>G@GH?G?F@?6<CB?B=DEC:>>CBE?G???BBG<.F:E?CFD?@?A:#5E>5BE/>BFFD+$E,>");

FastqSummaryReport report = new FastqSummaryReport();
report.parseRecord(rec);
assertEquals(1, report.getRecordsParsed());
assertTrue(report.instruments.isEmpty());
assertTrue(report.flowCellIds.isEmpty());
assertTrue(report.flowCellLanes.isEmpty());
assertTrue(report.tileNumbers.isEmpty());
assertEquals(0, report.firstInPair.intValue());
assertEquals(0, report.secondInPair.intValue());
}
}
Loading