| 1 | package edu.ucsb.cs156.courses.services; | |
| 2 | ||
| 3 | import com.opencsv.CSVReader; | |
| 4 | import com.opencsv.CSVReaderBuilder; | |
| 5 | import edu.ucsb.cs156.courses.entities.GradeHistory; | |
| 6 | import edu.ucsb.cs156.courses.utilities.CourseUtilities; | |
| 7 | import edu.ucsb.cs156.jobs.errors.JobCancelledException; | |
| 8 | import edu.ucsb.cs156.jobs.services.JobContext; | |
| 9 | import java.io.BufferedReader; | |
| 10 | import java.io.InputStreamReader; | |
| 11 | import java.sql.PreparedStatement; | |
| 12 | import java.sql.SQLException; | |
| 13 | import java.util.ArrayList; | |
| 14 | import java.util.HashMap; | |
| 15 | import java.util.List; | |
| 16 | import java.util.Map; | |
| 17 | import lombok.extern.slf4j.Slf4j; | |
| 18 | import org.springframework.beans.factory.annotation.Autowired; | |
| 19 | import org.springframework.http.HttpMethod; | |
| 20 | import org.springframework.jdbc.core.JdbcTemplate; | |
| 21 | import org.springframework.stereotype.Service; | |
| 22 | import org.springframework.web.client.RestTemplate; | |
| 23 | ||
| 24 | @Slf4j | |
| 25 | @Service | |
| 26 | public class GradeHistoryImportServiceImpl implements GradeHistoryImportService { | |
| 27 | ||
| 28 | public static class NullHeaderException extends RuntimeException { | |
| 29 | public NullHeaderException(String message) { | |
| 30 | super(message); | |
| 31 | } | |
| 32 | } | |
| 33 | ||
| 34 | @Autowired private JdbcTemplate jdbcTemplate; | |
| 35 | ||
| 36 | @Autowired private RestTemplate restTemplate; | |
| 37 | ||
| 38 | @Override | |
| 39 | public void importGradesFromUrl(String url, JobContext ctx, int batchSize) throws Exception { | |
| 40 | final int[] recordsProcessed = {0}; | |
| 41 | ||
| 42 | restTemplate.execute( | |
| 43 | url, | |
| 44 | HttpMethod.GET, | |
| 45 | null, | |
| 46 | response -> { | |
| 47 | try (BufferedReader reader = | |
| 48 | new BufferedReader(new InputStreamReader(response.getBody())); | |
| 49 | CSVReader csvReader = new CSVReaderBuilder(reader).build()) { | |
| 50 | ||
| 51 | String[] header = csvReader.readNext(); | |
| 52 |
1
1. lambda$importGradesFromUrl$0 : negated conditional → KILLED |
if (header == null) throw new NullHeaderException("CSV header is missing"); |
| 53 | ||
| 54 | Map<String, Integer> col = mapHeaders(header); | |
| 55 | List<GradeHistory> buffer = new ArrayList<>(); | |
| 56 | String[] nextLine; | |
| 57 | ||
| 58 | while ((nextLine = csvReader.readNext()) != null) { | |
| 59 | // ctx.log() below (and its cancellation check) only fires once per `batchSize` | |
| 60 | // rows -- checkCancellation() gives every row its own checkpoint, so a cancel | |
| 61 | // request doesn't have to wait for the next batch to flush on a very large CSV. | |
| 62 |
1
1. lambda$importGradesFromUrl$0 : removed call to edu/ucsb/cs156/jobs/services/JobContext::checkCancellation → KILLED |
ctx.checkCancellation(); |
| 63 | List<GradeHistory> gradesFromLine = mapLineToGrades(nextLine, col); | |
| 64 | buffer.addAll(gradesFromLine); | |
| 65 | ||
| 66 |
2
1. lambda$importGradesFromUrl$0 : changed conditional boundary → KILLED 2. lambda$importGradesFromUrl$0 : negated conditional → KILLED |
if (buffer.size() >= batchSize) { |
| 67 |
1
1. lambda$importGradesFromUrl$0 : removed call to edu/ucsb/cs156/courses/services/GradeHistoryImportServiceImpl::flushBuffer → KILLED |
flushBuffer(buffer, batchSize); |
| 68 |
1
1. lambda$importGradesFromUrl$0 : Replaced integer addition with subtraction → KILLED |
recordsProcessed[0] += buffer.size(); |
| 69 |
1
1. lambda$importGradesFromUrl$0 : removed call to edu/ucsb/cs156/jobs/services/JobContext::log → KILLED |
ctx.log("Processed " + recordsProcessed[0] + " grade history records so far."); |
| 70 |
1
1. lambda$importGradesFromUrl$0 : removed call to java/util/List::clear → KILLED |
buffer.clear(); |
| 71 | } | |
| 72 | } | |
| 73 |
1
1. lambda$importGradesFromUrl$0 : Replaced integer addition with subtraction → KILLED |
recordsProcessed[0] += buffer.size(); |
| 74 |
1
1. lambda$importGradesFromUrl$0 : removed call to edu/ucsb/cs156/jobs/services/JobContext::log → KILLED |
ctx.log("Processed " + recordsProcessed[0] + " grade history records. Done!"); |
| 75 |
1
1. lambda$importGradesFromUrl$0 : removed call to edu/ucsb/cs156/courses/services/GradeHistoryImportServiceImpl::flushBuffer → KILLED |
flushBuffer(buffer, batchSize); |
| 76 | ||
| 77 | } catch (NullHeaderException nhe) { | |
| 78 | log.error("Error processing CSV from URL: {}", url, nhe); | |
| 79 | throw nhe; | |
| 80 | } catch (JobCancelledException jce) { | |
| 81 | // Must not be wrapped: JobService's catch chain matches on this exact type to set | |
| 82 | // the job's terminal status to "cancelled" rather than "error". | |
| 83 | throw jce; | |
| 84 | } catch (Exception e) { | |
| 85 | log.error("Error processing CSV from URL: {}", url, e); | |
| 86 | throw new RuntimeException("CSV processing failed", e); | |
| 87 | } | |
| 88 | return null; | |
| 89 | }); | |
| 90 | } | |
| 91 | ||
| 92 | private Map<String, Integer> mapHeaders(String[] header) { | |
| 93 | Map<String, Integer> map = new HashMap<>(); | |
| 94 |
2
1. mapHeaders : changed conditional boundary → KILLED 2. mapHeaders : negated conditional → KILLED |
for (int i = 0; i < header.length; i++) { |
| 95 | map.put(header[i].trim(), i); | |
| 96 | } | |
| 97 |
1
1. mapHeaders : replaced return value with Collections.emptyMap for edu/ucsb/cs156/courses/services/GradeHistoryImportServiceImpl::mapHeaders → KILLED |
return map; |
| 98 | } | |
| 99 | ||
| 100 | private List<GradeHistory> mapLineToGrades(String[] line, Map<String, Integer> col) { | |
| 101 | List<GradeHistory> list = new ArrayList<>(); | |
| 102 | ||
| 103 | String year = line[col.get("year")]; | |
| 104 | String quarter = line[col.get("quarter")]; | |
| 105 | String yyyyq = year + CourseUtilities.quarterToDigit(quarter); | |
| 106 | String course = line[col.get("course")]; | |
| 107 | String instructor = line[col.get("instructor")]; | |
| 108 | ||
| 109 | // Map column names to cleaned Grade strings | |
| 110 | String[] gradeCols = { | |
| 111 | "Ap", "A", "Am", "Bp", "B", "Bm", "Cp", "C", "Cm", "Dp", "D", "Dm", "F", "P", "S" | |
| 112 | }; | |
| 113 | ||
| 114 | for (String grade : gradeCols) { | |
| 115 |
1
1. mapLineToGrades : negated conditional → KILLED |
if (col.containsKey(grade)) { |
| 116 | String val = line[col.get(grade)]; | |
| 117 | String convertedGrade = grade.replace("p", "+").replace("m", "-"); | |
| 118 |
1
1. mapLineToGrades : negated conditional → KILLED |
int count = (val.isEmpty()) ? 0 : Integer.parseInt(val); |
| 119 |
2
1. mapLineToGrades : negated conditional → KILLED 2. mapLineToGrades : changed conditional boundary → KILLED |
if (count > 0) { |
| 120 | list.add( | |
| 121 | GradeHistory.builder() | |
| 122 | .yyyyq(yyyyq) | |
| 123 | .course(course) | |
| 124 | .instructor(instructor) | |
| 125 | .grade(convertedGrade) | |
| 126 | .count(count) | |
| 127 | .build()); | |
| 128 | } | |
| 129 | } | |
| 130 | } | |
| 131 | ||
| 132 | int countNP = calculateNP(line, col); | |
| 133 |
2
1. mapLineToGrades : changed conditional boundary → KILLED 2. mapLineToGrades : negated conditional → KILLED |
if (countNP > 0) { |
| 134 | list.add( | |
| 135 | GradeHistory.builder() | |
| 136 | .yyyyq(yyyyq) | |
| 137 | .course(course) | |
| 138 | .instructor(instructor) | |
| 139 | .grade("NP") | |
| 140 | .count(countNP) | |
| 141 | .build()); | |
| 142 | } | |
| 143 |
1
1. mapLineToGrades : replaced return value with Collections.emptyList for edu/ucsb/cs156/courses/services/GradeHistoryImportServiceImpl::mapLineToGrades → KILLED |
return list; |
| 144 | } | |
| 145 | ||
| 146 | private int calculateNP(String[] line, Map<String, Integer> col) { | |
| 147 | String pVal = line[col.get("P")]; | |
| 148 | String nPnpVal = line[col.get("nPNPStudents")]; | |
| 149 | ||
| 150 |
1
1. calculateNP : negated conditional → KILLED |
int pCount = (pVal.isEmpty()) ? 0 : Integer.parseInt(pVal); |
| 151 |
1
1. calculateNP : negated conditional → KILLED |
int nPnpCount = (nPnpVal.isEmpty()) ? 0 : Integer.parseInt(nPnpVal); |
| 152 | ||
| 153 |
2
1. calculateNP : Replaced integer subtraction with addition → KILLED 2. calculateNP : replaced int return with 0 for edu/ucsb/cs156/courses/services/GradeHistoryImportServiceImpl::calculateNP → KILLED |
return nPnpCount - pCount; |
| 154 | } | |
| 155 | ||
| 156 | /** | |
| 157 | * This method flushes the buffer to the database in batches. It is public so that it can be spied | |
| 158 | * on using Mockito in the unit tests. | |
| 159 | * | |
| 160 | * @param buffer | |
| 161 | * @param batchSize | |
| 162 | */ | |
| 163 | public void flushBuffer(List<GradeHistory> buffer, int batchSize) { | |
| 164 | // Note: 'count' is excluded from the ON clause because it is the value we want | |
| 165 | // to update | |
| 166 | String sql = | |
| 167 | """ | |
| 168 | MERGE INTO "historygrade" AS t | |
| 169 | USING (VALUES (?, ?, ?, ?, ?)) AS s(yyyyq, course, instructor, grade, count) | |
| 170 | ON (t."yyyyq" = s.yyyyq AND t."course" = s.course AND t."instructor" = s.instructor AND t."grade" = s.grade) | |
| 171 | WHEN MATCHED THEN | |
| 172 | UPDATE SET "count" = s.count | |
| 173 | WHEN NOT MATCHED THEN | |
| 174 | INSERT ("yyyyq", "course", "instructor", "grade", "count") | |
| 175 | VALUES (s.yyyyq, s.course, s.instructor, s.grade, s.count); | |
| 176 | """; | |
| 177 | ||
| 178 | jdbcTemplate.batchUpdate(sql, buffer, batchSize, this::updateEntity); | |
| 179 | } | |
| 180 | ||
| 181 | public void updateEntity(PreparedStatement ps, GradeHistory entity) throws SQLException { | |
| 182 |
1
1. updateEntity : removed call to java/sql/PreparedStatement::setString → KILLED |
ps.setString(1, entity.getYyyyq()); |
| 183 |
1
1. updateEntity : removed call to java/sql/PreparedStatement::setString → KILLED |
ps.setString(2, entity.getCourse()); |
| 184 |
1
1. updateEntity : removed call to java/sql/PreparedStatement::setString → KILLED |
ps.setString(3, entity.getInstructor()); |
| 185 |
1
1. updateEntity : removed call to java/sql/PreparedStatement::setString → KILLED |
ps.setString(4, entity.getGrade()); |
| 186 |
1
1. updateEntity : removed call to java/sql/PreparedStatement::setInt → KILLED |
ps.setInt(5, entity.getCount()); |
| 187 | } | |
| 188 | } | |
Mutations | ||
| 52 |
1.1 |
|
| 62 |
1.1 |
|
| 66 |
1.1 2.2 |
|
| 67 |
1.1 |
|
| 68 |
1.1 |
|
| 69 |
1.1 |
|
| 70 |
1.1 |
|
| 73 |
1.1 |
|
| 74 |
1.1 |
|
| 75 |
1.1 |
|
| 94 |
1.1 2.2 |
|
| 97 |
1.1 |
|
| 115 |
1.1 |
|
| 118 |
1.1 |
|
| 119 |
1.1 2.2 |
|
| 133 |
1.1 2.2 |
|
| 143 |
1.1 |
|
| 150 |
1.1 |
|
| 151 |
1.1 |
|
| 153 |
1.1 2.2 |
|
| 182 |
1.1 |
|
| 183 |
1.1 |
|
| 184 |
1.1 |
|
| 185 |
1.1 |
|
| 186 |
1.1 |